mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-08 10:27:46 +08:00
Compare commits
321
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
911c7f8875 | ||
|
|
310098a853 | ||
|
|
466c7918a1 | ||
|
|
e7de935e61 | ||
|
|
b1943710b3 | ||
|
|
082348a5d1 | ||
|
|
ac7e3abab3 | ||
|
|
7acbaab82b | ||
|
|
716c35bf56 | ||
|
|
2539439656 | ||
|
|
e379bd791c | ||
|
|
84f0d5ce70 | ||
|
|
d4ed774423 | ||
|
|
0948f29da5 | ||
|
|
8a767c4309 | ||
|
|
bd35e885a7 | ||
|
|
8ffd8188e4 | ||
|
|
976c3feca6 | ||
|
|
5d5281b022 | ||
|
|
bda0d4e44d | ||
|
|
cb7b9c9ecd | ||
|
|
12d58327a1 | ||
|
|
2075cd95de | ||
|
|
625456ff24 | ||
|
|
e1dadf5b06 | ||
|
|
8f1e2decf2 | ||
|
|
9f829450d8 | ||
|
|
65f43a7aef | ||
|
|
2a63bafd20 | ||
|
|
ea6b739fd6 | ||
|
|
14befeda2c | ||
|
|
d13f52bdf8 | ||
|
|
2257c3959f | ||
|
|
066ea87d72 | ||
|
|
fae51a0ec8 | ||
|
|
db5d2dfbb5 | ||
|
|
54fbcc25a1 | ||
|
|
d068b62966 | ||
|
|
017d08d45e | ||
|
|
7f45617f79 | ||
|
|
9cc4018a37 | ||
|
|
b49f5c0fd7 | ||
|
|
125cd40aa5 | ||
|
|
491364867e | ||
|
|
8093899b5a | ||
|
|
fa06e9695d | ||
|
|
5d5880d75e | ||
|
|
9abbf268a1 | ||
|
|
e8ee7b4ecf | ||
|
|
c1aa5d618d | ||
|
|
309f507ef4 | ||
|
|
00a315e3bf | ||
|
|
f811e29f13 | ||
|
|
fb7e3fc224 | ||
|
|
cabca22d65 | ||
|
|
11989e278e | ||
|
|
532af797bf | ||
|
|
cafa05c4cb | ||
|
|
49ec53cbd9 | ||
|
|
73f1d79637 | ||
|
|
1b8f78a992 | ||
|
|
14ef19d9e1 | ||
|
|
bc0e9f94e2 | ||
|
|
74072e5007 | ||
|
|
c96f8272f4 | ||
|
|
4c07d9fcfb | ||
|
|
45678d9419 | ||
|
|
75bc32cfe9 | ||
|
|
d4bc058c2f | ||
|
|
2516e51b4e | ||
|
|
3541ccfe29 | ||
|
|
bd83cff58f | ||
|
|
85c04335c6 | ||
|
|
c08497c963 | ||
|
|
d30268f80f | ||
|
|
5679375f71 | ||
|
|
3465d23db3 | ||
|
|
e3333f1aef | ||
|
|
31d5e2d172 | ||
|
|
bb9f2eed8e | ||
|
|
926f5cc928 | ||
|
|
27ae884759 | ||
|
|
834eb9c308 | ||
|
|
390b73d4b6 | ||
|
|
bcd121d447 | ||
|
|
3394a51278 | ||
|
|
75471ae4a4 | ||
|
|
57f53903f5 | ||
|
|
81788d3a64 | ||
|
|
30bb0c3130 | ||
|
|
595b8e4e05 | ||
|
|
7e033d0571 | ||
|
|
5745442ed7 | ||
|
|
1a4eba1005 | ||
|
|
2a9d8d3b25 | ||
|
|
e25e240d16 | ||
|
|
cd765f2c2f | ||
|
|
ec95989e02 | ||
|
|
7f5e1a64fe | ||
|
|
f86dd10467 | ||
|
|
69930a6059 | ||
|
|
07cb401fd4 | ||
|
|
593327c803 | ||
|
|
a9a7c64e5d | ||
|
|
70d1a4ab74 | ||
|
|
0b7c7f94ac | ||
|
|
67d0414483 | ||
|
|
e3c01fb554 | ||
|
|
f960bbd2c8 | ||
|
|
906baae88e | ||
|
|
ba7c9f8b27 | ||
|
|
37e3a36680 | ||
|
|
166de33355 | ||
|
|
a95f0d2488 | ||
|
|
b296d46e97 | ||
|
|
0486435f16 | ||
|
|
fb25dde4c9 | ||
|
|
4124749a7d | ||
|
|
5a55116b62 | ||
|
|
681ce56c4f | ||
|
|
bcb2308000 | ||
|
|
6c92db2ba5 | ||
|
|
fdf55525f5 | ||
|
|
4ff4129034 | ||
|
|
364692da55 | ||
|
|
5842c7232e | ||
|
|
72a4bf3408 | ||
|
|
fe1723d87c | ||
|
|
03b198d5ab | ||
|
|
53562fd9de | ||
|
|
e66dd00b84 | ||
|
|
03496c46c5 | ||
|
|
c0ded116a9 | ||
|
|
e5ab73bf35 | ||
|
|
a5456cdc38 | ||
|
|
e9899200f8 | ||
|
|
5a6692ade0 | ||
|
|
6e431e2ff6 | ||
|
|
88df2a2ed5 | ||
|
|
e7864e5611 | ||
|
|
6e6407160a | ||
|
|
01acff0774 | ||
|
|
cc5050155f | ||
|
|
04c4a97766 | ||
|
|
e83399db2f | ||
|
|
f753f14fd2 | ||
|
|
7daf355e65 | ||
|
|
e4f89de90e | ||
|
|
dfe88e34e7 | ||
|
|
c5adcf0312 | ||
|
|
ea24d61910 | ||
|
|
23e0af7b12 | ||
|
|
60b89cc840 | ||
|
|
b748b5bfd2 | ||
|
|
30e36cd09a | ||
|
|
28cd77eb5e | ||
|
|
531f53b443 | ||
|
|
95e4d0149c | ||
|
|
72aea7898b | ||
|
|
8aedf87aad | ||
|
|
e9b64c3e9d | ||
|
|
28f61ec45b | ||
|
|
6aeadcd1d7 | ||
|
|
3a8dadcd6b | ||
|
|
ecc16673eb | ||
|
|
d28dd89039 | ||
|
|
33ea4ebf15 | ||
|
|
8260a87215 | ||
|
|
361952ada9 | ||
|
|
6630856061 | ||
|
|
a893bd0557 | ||
|
|
f2839ae6a7 | ||
|
|
e58570d79d | ||
|
|
99f6499b2b | ||
|
|
17d01d7fe0 | ||
|
|
8b766930b0 | ||
|
|
c7e403b410 | ||
|
|
cf8ea19856 | ||
|
|
7113d04f8a | ||
|
|
099b810a2f | ||
|
|
7aa0c89244 | ||
|
|
7847ae98c6 | ||
|
|
a90d564931 | ||
|
|
a5c3699ae9 | ||
|
|
7b8048c6ae | ||
|
|
ec95f2ca1f | ||
|
|
aa7dbe67d3 | ||
|
|
a26680f460 | ||
|
|
522b979052 | ||
|
|
808946312a | ||
|
|
741107bf71 | ||
|
|
6962731220 | ||
|
|
062e111c03 | ||
|
|
470c59e197 | ||
|
|
2f929e74c7 | ||
|
|
fc0417ceb9 | ||
|
|
44174a31e0 | ||
|
|
b599fb7354 | ||
|
|
14f96c9fa0 | ||
|
|
6948852992 | ||
|
|
1b01b08c31 | ||
|
|
2281f2b754 | ||
|
|
b5ed802277 | ||
|
|
d1b5eb08ee | ||
|
|
dba5e6e9e9 | ||
|
|
c125e78c5f | ||
|
|
d1cb0ebecf | ||
|
|
9d7a0665c0 | ||
|
|
882bb43125 | ||
|
|
db6c522d60 | ||
|
|
e29442a06a | ||
|
|
e15ea0d5d3 | ||
|
|
2f374d6af2 | ||
|
|
4a356f4ea5 | ||
|
|
c7676d567d | ||
|
|
5ca4f87951 | ||
|
|
1fee8954cc | ||
|
|
f69b770f5e | ||
|
|
856accdced | ||
|
|
92749b4d6e | ||
|
|
f08c2e6729 | ||
|
|
e420bc6324 | ||
|
|
d723fb92d3 | ||
|
|
5b1de5f921 | ||
|
|
7ed48e7b58 | ||
|
|
af712ebdbf | ||
|
|
33d5cd5993 | ||
|
|
f5e1420ee6 | ||
|
|
b37b252b14 | ||
|
|
0097ea89ad | ||
|
|
10e63507f0 | ||
|
|
9ff4d73d5c | ||
|
|
0e3bd7eff4 | ||
|
|
1c89b5f9ab | ||
|
|
cdbbda40a6 | ||
|
|
29a9d608d9 | ||
|
|
a6dc43d5f6 | ||
|
|
c6718754d3 | ||
|
|
afdd033745 | ||
|
|
d5f54ffe8b | ||
|
|
f5ec76c5c8 | ||
|
|
784a1e0611 | ||
|
|
507cb33089 | ||
|
|
b08fa3bdb6 | ||
|
|
018af84d7d | ||
|
|
27b0381a9a | ||
|
|
57cdef4b8d | ||
|
|
36e9d21e3f | ||
|
|
b72b6ab137 | ||
|
|
30b2c8548a | ||
|
|
7c5cce4b3c | ||
|
|
9362c34fcd | ||
|
|
344b3031e9 | ||
|
|
e89c3aa674 | ||
|
|
ddbbf835af | ||
|
|
cb58a63ee3 | ||
|
|
6b1074cfcd | ||
|
|
d6894b5532 | ||
|
|
635c6765d9 | ||
|
|
86f7cc0d58 | ||
|
|
206995645b | ||
|
|
9282cce1d6 | ||
|
|
c005700a7e | ||
|
|
14744abd57 | ||
|
|
66d6c17d2d | ||
|
|
c142d39951 | ||
|
|
1eb2d10dec | ||
|
|
dabaeb8dfa | ||
|
|
2d17d4b73f | ||
|
|
18d78dd6c9 | ||
|
|
499942e3e7 | ||
|
|
ba11a72214 | ||
|
|
12571764bc | ||
|
|
1e13fa032c | ||
|
|
47b21a25d3 | ||
|
|
45a3ba8829 | ||
|
|
03f2914044 | ||
|
|
c8d1ae3e7e | ||
|
|
c5ae9c2c77 | ||
|
|
4e47c00154 | ||
|
|
313a637982 | ||
|
|
fe8ff268df | ||
|
|
bac6d6866a | ||
|
|
579f2c7cc1 | ||
|
|
ddcbeb3ae9 | ||
|
|
e25fc984af | ||
|
|
4cf47b1dee | ||
|
|
95cbd43097 | ||
|
|
09005939bf | ||
|
|
7b612b8b5a | ||
|
|
670d5e8d33 | ||
|
|
1de2e70d41 | ||
|
|
89b57464d2 | ||
|
|
09ef3adf70 | ||
|
|
3dfc15963c | ||
|
|
f6884eb8c4 | ||
|
|
f8b4382a54 | ||
|
|
d78b5a81fb | ||
|
|
89fe9e9f0a | ||
|
|
4291a91dc0 | ||
|
|
4c6bafe255 | ||
|
|
979dbc4b33 | ||
|
|
9309ad844f | ||
|
|
d672ba2068 | ||
|
|
587486ab0c | ||
|
|
40a5e1470d | ||
|
|
058660ec2e | ||
|
|
668bf5e40f | ||
|
|
77f93c638d | ||
|
|
d0c0996b9f | ||
|
|
76fb8905c9 | ||
|
|
f822df6cce | ||
|
|
45c840b8d3 | ||
|
|
214f3d6406 | ||
|
|
cc6f5e89b6 | ||
|
|
2ed2cc66ef | ||
|
|
b1bf7837cf | ||
|
|
77229943d1 | ||
|
|
a0369cf49a | ||
|
|
dbbe7b22ab | ||
|
|
d5f34b2ee2 |
@@ -0,0 +1,6 @@
|
|||||||
|
# 不固定 test-threads:nextest 默认按 num-cpus 并发,固定值会在更大规格的
|
||||||
|
# runner 或本地开发机上主动压低并发、反而变慢,且无法表达 min(4, num-cpus)。
|
||||||
|
# 这里只保留卡死保护,避免单个挂起用例拖满整个 job。
|
||||||
|
[profile.default]
|
||||||
|
# 60 秒后标记慢测试,连续两轮仍未结束则终止;超时结果保持失败,不隐藏回归。
|
||||||
|
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||||
+96
-8
@@ -26,21 +26,25 @@ RUST_LOG=aether_gateway=info
|
|||||||
# 示例: http://localhost:5173,https://app.example.com
|
# 示例: http://localhost:5173,https://app.example.com
|
||||||
# CORS_ORIGINS=http://localhost:5173
|
# CORS_ORIGINS=http://localhost:5173
|
||||||
# CORS_ALLOW_CREDENTIALS=true
|
# CORS_ALLOW_CREDENTIALS=true
|
||||||
# 如果前后端跨站并依赖登录刷新 Cookie,还要配合:
|
# 登录刷新 Cookie 对同源浏览器请求和可信反代自动适配 HTTP/HTTPS。
|
||||||
|
# HTTP 自动使用兼容的 SameSite=Lax(显式 Strict 保留);HTTPS 保留原有 SameSite 配置。
|
||||||
|
# 无法确认访问协议时保留安全默认值;HTTPS 反代请正确传递 X-Forwarded-Proto。
|
||||||
|
# AUTH_REFRESH_COOKIE_SECURE 可显式覆盖自动判断,公网部署仍建议使用 HTTPS。
|
||||||
|
# 如果前后端跨站并依赖登录刷新 Cookie,必须使用 HTTPS,并配合:
|
||||||
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
||||||
# AUTH_REFRESH_COOKIE_SECURE=true
|
# AUTH_REFRESH_COOKIE_SECURE=true
|
||||||
|
|
||||||
# 数据库配置
|
# 数据库配置(仅支持 PostgreSQL)
|
||||||
DB_HOST=localhost
|
DB_HOST=localhost
|
||||||
DB_PORT=5432
|
DB_PORT=5432
|
||||||
DB_USER=postgres
|
DB_USER=postgres
|
||||||
DB_NAME=aether
|
DB_NAME=aether
|
||||||
DB_PASSWORD=aether
|
DB_PASSWORD=
|
||||||
|
|
||||||
# Redis 配置
|
# Redis 配置
|
||||||
REDIS_HOST=localhost
|
REDIS_HOST=localhost
|
||||||
REDIS_PORT=6379
|
REDIS_PORT=6379
|
||||||
REDIS_PASSWORD=aether
|
REDIS_PASSWORD=
|
||||||
|
|
||||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||||
@@ -50,6 +54,10 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
|||||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||||
|
|
||||||
|
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||||
|
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||||
|
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||||
|
|
||||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||||
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||||
ADMIN_EMAIL=[email protected]
|
ADMIN_EMAIL=[email protected]
|
||||||
@@ -75,15 +83,95 @@ ADMIN_USERNAME=admin123456
|
|||||||
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
||||||
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
||||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||||
|
# 普通 PostgreSQL 连接默认语句超时 30 秒、锁等待超时 3 秒;0 显式关闭。
|
||||||
|
# 这是单条 SQL 的期限,不是整个事务总期限;迁移和历史 backfill 使用独立连接放宽。
|
||||||
|
# AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS=30000
|
||||||
|
# AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS=3000
|
||||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||||
|
# 所有监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和升级后的 WebSocket。
|
||||||
|
# 未设置或 0 时按请求上限 + WebSocket 上限推导,最大 65536;显式配置也受 FD 余量限制。
|
||||||
|
# 已知 FD soft limit 时最多 max(1, (FD - 256) / 2),不是整个进程的 FD/内存保证。
|
||||||
|
# 满额的新连接在 HTTP 解析前关闭,不排队创建任务,也不会返回 HTTP 429/503。
|
||||||
|
# AETHER_GATEWAY_MAX_HTTP_CONNECTIONS=4096
|
||||||
|
# 停机先等待 HTTP 请求,再排空本地用量写入;以下期限单位为毫秒。
|
||||||
|
# 进程管理器的强杀期限应覆盖两阶段之和,再预留至少 10 秒收尾。
|
||||||
|
# AETHER_GATEWAY_HTTP_SHUTDOWN_TIMEOUT_MS=30000
|
||||||
|
# AETHER_GATEWAY_USAGE_SHUTDOWN_TIMEOUT_MS=30000
|
||||||
|
# 每客户端、每 origin 的上游空闲连接缓存;不限制活动请求或流持续时间。
|
||||||
|
# AETHER_GATEWAY_UPSTREAM_POOL_MAX_IDLE_PER_HOST=32
|
||||||
|
# AETHER_GATEWAY_UPSTREAM_POOL_IDLE_TIMEOUT_MS=15000
|
||||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||||
|
# 请求体按实际缓冲增长申请额度,解压同时计入输入和输出;额度不足返回 503。
|
||||||
|
# 请求体完整读取总超时默认 120000 毫秒;非零值限制在 1000-600000,显式 0 关闭。
|
||||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
||||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
# 上游流首包后空闲超时默认 300000 毫秒;执行配置 read_ms 优先,显式 0 关闭。
|
||||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
# AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS=300000
|
||||||
|
# 流式响应诊断捕获共享预算默认 128 MiB,包含 provider/client 分配;不足时截断审计副本。
|
||||||
|
# 显式 0 关闭此类捕获;不限制协议解析、终态编码和 usage 队列的总内存。
|
||||||
|
# AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||||
|
# usage 诊断正文共享预算,按 JSON 堆内存估算,默认 128 MiB。
|
||||||
|
# 覆盖终态队列 seed、Redis 解码事件、数据库写入 DTO 及正文副本;额度随正文释放。
|
||||||
|
# 不足或显式 0 时先保留计费事实再舍弃正文;已有清空/禁用状态保留,其余标为截断。
|
||||||
|
# 不包含原始 Redis 批次、解码临时分配、序列化和压缩结果、协议观察缓冲或进程总内存。
|
||||||
|
# AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||||
|
# 新 usage 队列消息的完整 JSON payload 上限,默认 1 MiB;显式 0 非法。
|
||||||
|
# 超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义则拒绝入队,终态尝试受限落库,失败即明确失败。
|
||||||
|
# 不限制存量 Redis 消息、整个读取批次、DLQ 或进程总内存。
|
||||||
|
# usage_runtime_queue_payload_* 降级/拒绝计数包含入队和重试预校验的编码尝试,不代表唯一事件数。
|
||||||
|
# AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES=1048576
|
||||||
|
# usage worker 读取/重领共用的逻辑 payload 预留:全进程默认 128 MiB,单批目标 8 MiB。
|
||||||
|
# 按当前消息上限推导 COUNT,默认最多 8 条;预留覆盖整批处理和确认,额度不足等待。
|
||||||
|
# 当前消息上限不能超过总预留额度;单批目标不足一条时仍读一条。0 或非法值回退默认。
|
||||||
|
# 历史/其他生产者的大消息继续处理并计数,不是 RESP、实际堆内存或 DLQ 的硬上限。
|
||||||
|
# AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES=134217728
|
||||||
|
# AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES=8388608
|
||||||
|
# DLQ 原文和最坏 JSON 编码独立预留默认 64 MiB,后台编码及写入最多同时 4 个任务。
|
||||||
|
# 入场前一次预留,额度占满或单条超预算立即失败并保留 pending 原消息,后续重领。
|
||||||
|
# 编码失败不阻塞同批其余正常消息;批次仍报告失败,只有成功项会被确认。
|
||||||
|
# JSON 按字符串最多 6 倍转义保守估算;不截原账务字段,不包含 Redis 命令/连接副本或 RSS。
|
||||||
|
# 0/非法值回退默认;bytes 最大约 4 GiB,jobs 最大 128。超大存量可能需调高额度后恢复。
|
||||||
|
# AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES=67108864
|
||||||
|
# AETHER_USAGE_DLQ_ENCODING_MAX_JOBS=4
|
||||||
|
# 内置 Redis 死信转移要求 Redis 7+ 及 EVAL/TYPE/XPENDING/XADD/XACK/XDEL 权限。
|
||||||
|
# stream 与 DLQ 不能同名;Cluster 还要求两键同 slot,现有默认键未自动迁移。
|
||||||
|
# 单请求解压后 Payload 上限(MiB),默认 256;显式 0 仍受 256 MiB 硬上限保护。
|
||||||
|
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
|
||||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
|
||||||
|
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
|
||||||
|
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
|
||||||
|
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||||
|
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||||
|
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||||
|
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||||
|
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||||
|
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||||
|
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||||
|
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||||
|
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||||
|
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||||
|
# 跨网关 relay 解析到受控私有地址时才显式开启;默认关闭以防止被篡改的 attachment
|
||||||
|
# 记录诱导网关向内网转发 relay 凭据。该开关不放宽普通 provider 的目标地址策略。
|
||||||
|
# AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS=false
|
||||||
|
# 更推荐按 relay 主机名精确放行私网部署(逗号分隔,大小写不敏感);不支持通配符/后缀。
|
||||||
|
# AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST=gateway-a.internal,gateway-b.internal
|
||||||
|
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||||
|
# AETHER_BARK_ALLOW_HTTP=false
|
||||||
|
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||||
|
|
||||||
|
# 普通 Provider 反代(包括 Provider OAuth)不按 DNS 地址过滤上游,兼容任意
|
||||||
|
# Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。
|
||||||
|
# URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。
|
||||||
|
|
||||||
|
# 可选 Provider OAuth 客户端。Gemini CLI 和 Antigravity 默认使用内置 native-app
|
||||||
|
# 客户端凭据;自定义 client ID 时必须同时配置对应的 client secret。
|
||||||
|
# 显式配置的 client secret 优先于默认值。
|
||||||
|
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||||
|
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||||
|
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||||
|
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||||
|
|
||||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
actions: read
|
||||||
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: build-tunnel-${{ github.ref }}
|
group: build-tunnel-${{ github.ref }}
|
||||||
@@ -17,7 +18,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Ensure tunnel tag matches Cargo version
|
- name: Ensure tunnel tag matches Cargo version
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -78,10 +79,10 @@ jobs:
|
|||||||
use_cross: false
|
use_cross: false
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
@@ -89,14 +90,14 @@ jobs:
|
|||||||
run: rustup target add ${{ matrix.target }}
|
run: rustup target add ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
workspaces: apps/aether-tunnel -> target
|
workspaces: apps/aether-tunnel -> target
|
||||||
key: ${{ matrix.target }}
|
key: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
working-directory: apps/aether-tunnel
|
working-directory: apps/aether-tunnel
|
||||||
@@ -122,9 +123,10 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
cd target/${{ matrix.target }}/release
|
cd target/${{ matrix.target }}/release
|
||||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||||
|
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||||
|
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-tunnel-${{ matrix.name }}
|
name: aether-tunnel-${{ matrix.name }}
|
||||||
path: |
|
path: |
|
||||||
@@ -137,9 +139,14 @@ jobs:
|
|||||||
needs: build
|
needs: build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: write
|
||||||
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
merge-multiple: true
|
merge-multiple: true
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -148,6 +155,20 @@ jobs:
|
|||||||
working-directory: artifacts
|
working-directory: artifacts
|
||||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||||
|
|
||||||
|
- name: Attest tunnel release provenance
|
||||||
|
id: attest-release
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-path: |
|
||||||
|
artifacts/aether-tunnel-*.tar.gz
|
||||||
|
artifacts/aether-tunnel-*.zip
|
||||||
|
artifacts/SHA256SUMS.txt
|
||||||
|
|
||||||
|
- name: Bundle tunnel release provenance
|
||||||
|
env:
|
||||||
|
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||||
|
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||||
|
|
||||||
- name: Delete stale draft releases for tag
|
- name: Delete stale draft releases for tag
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
@@ -170,12 +191,13 @@ jobs:
|
|||||||
done <<< "${draft_ids}"
|
done <<< "${draft_ids}"
|
||||||
|
|
||||||
- name: Create GitHub Release
|
- name: Create GitHub Release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||||
with:
|
with:
|
||||||
name: "${{ github.ref_name }}"
|
name: "${{ github.ref_name }}"
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
files: |
|
files: |
|
||||||
artifacts/aether-tunnel-*
|
artifacts/aether-tunnel-*
|
||||||
|
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||||
artifacts/SHA256SUMS.txt
|
artifacts/SHA256SUMS.txt
|
||||||
fail_on_unmatched_files: true
|
fail_on_unmatched_files: true
|
||||||
|
|
||||||
@@ -183,8 +205,10 @@ jobs:
|
|||||||
needs: release
|
needs: release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: main
|
ref: main
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ on:
|
|||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
pages: write
|
|
||||||
id-token: write
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: pages
|
group: pages
|
||||||
@@ -46,10 +44,10 @@ jobs:
|
|||||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
cache: 'npm'
|
||||||
@@ -77,10 +75,10 @@ jobs:
|
|||||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||||
|
|
||||||
- name: Setup Pages
|
- name: Setup Pages
|
||||||
uses: actions/configure-pages@v5
|
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||||
|
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
uses: actions/upload-pages-artifact@v3
|
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||||
with:
|
with:
|
||||||
path: frontend/dist
|
path: frontend/dist
|
||||||
|
|
||||||
@@ -90,7 +88,10 @@ jobs:
|
|||||||
url: ${{ steps.deployment.outputs.page_url }}
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
pages: write
|
||||||
steps:
|
steps:
|
||||||
- name: Deploy to GitHub Pages
|
- name: Deploy to GitHub Pages
|
||||||
id: deployment
|
id: deployment
|
||||||
uses: actions/deploy-pages@v4
|
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ env:
|
|||||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||||
CARGO_TERM_COLOR: always
|
CARGO_TERM_COLOR: always
|
||||||
RUST_BACKTRACE: '1'
|
RUST_BACKTRACE: '1'
|
||||||
GHCR_IMAGE: ghcr.io/fawney19/aether
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
source:
|
source:
|
||||||
@@ -36,6 +35,7 @@ jobs:
|
|||||||
sha: ${{ steps.snapshot.outputs.sha }}
|
sha: ${{ steps.snapshot.outputs.sha }}
|
||||||
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
||||||
date: ${{ steps.snapshot.outputs.date }}
|
date: ${{ steps.snapshot.outputs.date }}
|
||||||
|
ghcr_image: ${{ steps.snapshot.outputs.ghcr_image }}
|
||||||
steps:
|
steps:
|
||||||
- name: Require main branch
|
- name: Require main branch
|
||||||
id: snapshot
|
id: snapshot
|
||||||
@@ -49,9 +49,13 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
sha="${GITHUB_SHA}"
|
sha="${GITHUB_SHA}"
|
||||||
|
# Docker 镜像仓库名必须全小写;GitHub owner 可能保留大写,先统一规范化。
|
||||||
|
repository_owner="${GITHUB_REPOSITORY%%/*}"
|
||||||
|
repository_owner="${repository_owner,,}"
|
||||||
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
||||||
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
||||||
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
||||||
|
echo "ghcr_image=ghcr.io/${repository_owner}/aether" >> "${GITHUB_OUTPUT}"
|
||||||
echo "Building main at ${sha}."
|
echo "Building main at ${sha}."
|
||||||
|
|
||||||
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
||||||
@@ -59,6 +63,8 @@ jobs:
|
|||||||
name: Rust CI
|
name: Rust CI
|
||||||
needs: source
|
needs: source
|
||||||
uses: ./.github/workflows/rust-ci.yml
|
uses: ./.github/workflows/rust-ci.yml
|
||||||
|
with:
|
||||||
|
full_scope: true
|
||||||
|
|
||||||
rust_extended:
|
rust_extended:
|
||||||
name: Rust extended checks
|
name: Rust extended checks
|
||||||
@@ -66,12 +72,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 90
|
timeout-minutes: 90
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Install pinned Rust toolchain
|
- name: Install pinned Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
|
|
||||||
@@ -79,13 +85,13 @@ jobs:
|
|||||||
run: rustc -Vv
|
run: rustc -Vv
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Check all workspace targets
|
- name: Check all workspace targets
|
||||||
env:
|
env:
|
||||||
@@ -112,12 +118,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -157,7 +163,7 @@ jobs:
|
|||||||
run: npm run build
|
run: npm run build
|
||||||
|
|
||||||
- name: Upload frontend artifact
|
- name: Upload frontend artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-frontend-dist
|
name: nightly-frontend-dist
|
||||||
path: frontend/dist/
|
path: frontend/dist/
|
||||||
@@ -171,12 +177,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
|
|
||||||
@@ -247,38 +253,26 @@ jobs:
|
|||||||
arch: arm64
|
arch: arm64
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
use_cross: true
|
use_cross: true
|
||||||
- name: macos-amd64
|
|
||||||
target: x86_64-apple-darwin
|
|
||||||
platform: macos
|
|
||||||
arch: amd64
|
|
||||||
os: macos-15-intel
|
|
||||||
use_cross: false
|
|
||||||
- name: macos-arm64
|
|
||||||
target: aarch64-apple-darwin
|
|
||||||
platform: macos
|
|
||||||
arch: arm64
|
|
||||||
os: macos-15
|
|
||||||
use_cross: false
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Install pinned Rust toolchain
|
- name: Install pinned Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: nightly-release-${{ matrix.target }}
|
shared-key: nightly-release-${{ matrix.target }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build release binary
|
- name: Build release binary
|
||||||
env:
|
env:
|
||||||
@@ -295,7 +289,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload binary artifact
|
- name: Upload binary artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||||
path: target/${{ matrix.target }}/release/aether-gateway
|
path: target/${{ matrix.target }}/release/aether-gateway
|
||||||
@@ -308,17 +302,19 @@ jobs:
|
|||||||
needs: [source, checks, build]
|
needs: [source, checks, build]
|
||||||
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||||
permissions:
|
permissions:
|
||||||
actions: read
|
actions: read
|
||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Download Linux binaries and frontend
|
- name: Download Linux binaries and frontend
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
pattern: nightly-*
|
pattern: nightly-*
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -335,20 +331,20 @@ jobs:
|
|||||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Build and push nightly image
|
- name: Build and push nightly image
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile.app
|
file: ./Dockerfile.app
|
||||||
@@ -372,12 +368,12 @@ jobs:
|
|||||||
actions: read
|
actions: read
|
||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Download nightly artifacts
|
- name: Download nightly artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
pattern: nightly-*
|
pattern: nightly-*
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -392,31 +388,29 @@ jobs:
|
|||||||
VERSION="nightly"
|
VERSION="nightly"
|
||||||
|
|
||||||
mkdir -p package release-assets
|
mkdir -p package release-assets
|
||||||
for platform in linux macos; do
|
for arch in amd64 arm64; do
|
||||||
for arch in amd64 arm64; do
|
bundle="aether-${VERSION}-linux-${arch}"
|
||||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
root="package/${bundle}"
|
||||||
root="package/${bundle}"
|
mkdir -p "${root}/bin" "${root}/frontend"
|
||||||
mkdir -p "${root}/bin" "${root}/frontend"
|
|
||||||
|
|
||||||
install -m 0755 \
|
install -m 0755 \
|
||||||
"artifacts/nightly-gateway-${platform}-${arch}/aether-gateway" \
|
"artifacts/nightly-gateway-linux-${arch}/aether-gateway" \
|
||||||
"${root}/bin/aether-gateway"
|
"${root}/bin/aether-gateway"
|
||||||
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
||||||
sed \
|
sed \
|
||||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||||
install.sh > "${root}/install.sh"
|
install.sh > "${root}/install.sh"
|
||||||
chmod 0755 "${root}/install.sh"
|
chmod 0755 "${root}/install.sh"
|
||||||
install -m 0755 update.sh "${root}/update.sh"
|
install -m 0755 update.sh "${root}/update.sh"
|
||||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||||
install -m 0644 .env.example "${root}/.env.example"
|
install -m 0644 .env.example "${root}/.env.example"
|
||||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||||
install -m 0644 README.md "${root}/README.md"
|
install -m 0644 README.md "${root}/README.md"
|
||||||
install -m 0644 LICENSE "${root}/LICENSE"
|
install -m 0644 LICENSE "${root}/LICENSE"
|
||||||
|
|
||||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||||
done
|
|
||||||
done
|
done
|
||||||
|
|
||||||
sed \
|
sed \
|
||||||
@@ -426,15 +420,15 @@ jobs:
|
|||||||
chmod 0755 release-assets/install.sh
|
chmod 0755 release-assets/install.sh
|
||||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||||
|
|
||||||
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 4
|
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 2
|
||||||
test "$(wc -l < release-assets/SHA256SUMS)" -eq 4
|
test "$(wc -l < release-assets/SHA256SUMS)" -eq 2
|
||||||
(cd release-assets && sha256sum -c SHA256SUMS)
|
(cd release-assets && sha256sum -c SHA256SUMS)
|
||||||
for archive in release-assets/*.tar.gz; do
|
for archive in release-assets/*.tar.gz; do
|
||||||
tar -tzf "${archive}" >/dev/null
|
tar -tzf "${archive}" >/dev/null
|
||||||
done
|
done
|
||||||
|
|
||||||
- name: Upload nightly package artifact
|
- name: Upload nightly package artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-release-assets
|
name: nightly-release-assets
|
||||||
path: release-assets/*
|
path: release-assets/*
|
||||||
@@ -452,7 +446,7 @@ jobs:
|
|||||||
contents: write
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download nightly package artifact
|
- name: Download nightly package artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-release-assets
|
name: nightly-release-assets
|
||||||
path: release-assets
|
path: release-assets
|
||||||
@@ -466,6 +460,7 @@ jobs:
|
|||||||
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
||||||
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
||||||
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
||||||
|
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
@@ -510,6 +505,15 @@ jobs:
|
|||||||
--repo "${REPOSITORY}" \
|
--repo "${REPOSITORY}" \
|
||||||
--clobber
|
--clobber
|
||||||
|
|
||||||
|
published_assets="$(gh release view "${RELEASE_TAG}" --repo "${REPOSITORY}" --json assets --jq '.assets[].name')"
|
||||||
|
while IFS= read -r asset_name; do
|
||||||
|
if [[ "${asset_name}" == aether-nightly-*.tar.gz && ! -f "release-assets/${asset_name}" ]]; then
|
||||||
|
gh release delete-asset "${RELEASE_TAG}" "${asset_name}" \
|
||||||
|
--repo "${REPOSITORY}" \
|
||||||
|
--yes
|
||||||
|
fi
|
||||||
|
done <<<"${published_assets}"
|
||||||
|
|
||||||
# target_commitish does not move an existing git tag. Move the ref
|
# target_commitish does not move an existing git tag. Move the ref
|
||||||
# only after the complete asset set is available.
|
# only after the complete asset set is available.
|
||||||
if gh api "repos/${REPOSITORY}/git/ref/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
|
if gh api "repos/${REPOSITORY}/git/ref/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
|
||||||
@@ -534,8 +538,6 @@ jobs:
|
|||||||
expected_assets=(
|
expected_assets=(
|
||||||
aether-nightly-linux-amd64.tar.gz
|
aether-nightly-linux-amd64.tar.gz
|
||||||
aether-nightly-linux-arm64.tar.gz
|
aether-nightly-linux-arm64.tar.gz
|
||||||
aether-nightly-macos-amd64.tar.gz
|
|
||||||
aether-nightly-macos-arm64.tar.gz
|
|
||||||
SHA256SUMS
|
SHA256SUMS
|
||||||
install.sh
|
install.sh
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
actions: read
|
||||||
packages: write
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: release-aether-${{ github.ref }}
|
group: release-aether-${{ github.ref }}
|
||||||
@@ -70,10 +70,10 @@ jobs:
|
|||||||
needs: preflight
|
needs: preflight
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||||
with:
|
with:
|
||||||
node-version: 22
|
node-version: 22
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -94,7 +94,7 @@ jobs:
|
|||||||
npm run build
|
npm run build
|
||||||
|
|
||||||
- name: Upload frontend artifact
|
- name: Upload frontend artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: frontend-dist
|
name: frontend-dist
|
||||||
path: frontend/dist/
|
path: frontend/dist/
|
||||||
@@ -106,10 +106,10 @@ jobs:
|
|||||||
needs: preflight
|
needs: preflight
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||||
with:
|
with:
|
||||||
node-version: 22
|
node-version: 22
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -159,7 +159,7 @@ jobs:
|
|||||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||||
|
|
||||||
- name: Upload VSIX artifact
|
- name: Upload VSIX artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-vscodex-vsix
|
name: aether-vscodex-vsix
|
||||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||||
@@ -186,35 +186,23 @@ jobs:
|
|||||||
arch: arm64
|
arch: arm64
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
use_cross: true
|
use_cross: true
|
||||||
- name: macos-amd64
|
|
||||||
target: x86_64-apple-darwin
|
|
||||||
platform: macos
|
|
||||||
arch: amd64
|
|
||||||
os: macos-15-intel
|
|
||||||
use_cross: false
|
|
||||||
- name: macos-arm64
|
|
||||||
target: aarch64-apple-darwin
|
|
||||||
platform: macos
|
|
||||||
arch: arm64
|
|
||||||
os: macos-15
|
|
||||||
use_cross: false
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: release-${{ matrix.target }}
|
shared-key: release-${{ matrix.target }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
env:
|
env:
|
||||||
@@ -230,7 +218,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload binary artifact
|
- name: Upload binary artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||||
path: target/${{ matrix.target }}/release/aether-gateway
|
path: target/${{ matrix.target }}/release/aether-gateway
|
||||||
@@ -242,11 +230,17 @@ jobs:
|
|||||||
needs: [preflight, frontend, build]
|
needs: [preflight, frontend, build]
|
||||||
if: needs.preflight.outputs.publish == 'true'
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
|
|
||||||
@@ -259,27 +253,27 @@ jobs:
|
|||||||
cp -r artifacts/frontend-dist dist/frontend
|
cp -r artifacts/frontend-dist dist/frontend
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Log in to Docker Hub
|
- name: Log in to Docker Hub
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Extract metadata
|
- name: Extract metadata
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v5
|
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||||
with:
|
with:
|
||||||
images: |
|
images: |
|
||||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||||
@@ -295,7 +289,8 @@ jobs:
|
|||||||
latest=false
|
latest=false
|
||||||
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
uses: docker/build-push-action@v6
|
id: push
|
||||||
|
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile.app
|
file: ./Dockerfile.app
|
||||||
@@ -304,15 +299,36 @@ jobs:
|
|||||||
labels: ${{ steps.meta.outputs.labels }}
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
|
|
||||||
|
- name: Attest GHCR image provenance
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||||
|
subject-digest: ${{ steps.push.outputs.digest }}
|
||||||
|
push-to-registry: true
|
||||||
|
create-storage-record: false
|
||||||
|
|
||||||
|
- name: Attest Docker Hub image provenance
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||||
|
subject-digest: ${{ steps.push.outputs.digest }}
|
||||||
|
push-to-registry: true
|
||||||
|
create-storage-record: false
|
||||||
|
|
||||||
package:
|
package:
|
||||||
name: Release tarballs
|
name: Release tarballs
|
||||||
needs: [preflight, frontend, build]
|
needs: [preflight, frontend, build]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
|
|
||||||
@@ -328,31 +344,29 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p package release-assets
|
mkdir -p package release-assets
|
||||||
for platform in linux macos; do
|
for arch in amd64 arm64; do
|
||||||
for arch in amd64 arm64; do
|
bundle="aether-${VERSION}-linux-${arch}"
|
||||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
root="package/${bundle}"
|
||||||
root="package/${bundle}"
|
mkdir -p \
|
||||||
mkdir -p \
|
"${root}/bin" \
|
||||||
"${root}/bin" \
|
"${root}/frontend"
|
||||||
"${root}/frontend"
|
|
||||||
|
|
||||||
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
install -m 0755 "artifacts/aether-gateway-linux-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||||
sed \
|
sed \
|
||||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||||
install.sh > "${root}/install.sh"
|
install.sh > "${root}/install.sh"
|
||||||
chmod 0755 "${root}/install.sh"
|
chmod 0755 "${root}/install.sh"
|
||||||
install -m 0755 update.sh "${root}/update.sh"
|
install -m 0755 update.sh "${root}/update.sh"
|
||||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||||
install -m 0644 .env.example "${root}/.env.example"
|
install -m 0644 .env.example "${root}/.env.example"
|
||||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||||
install -m 0644 README.md "${root}/README.md"
|
install -m 0644 README.md "${root}/README.md"
|
||||||
install -m 0644 LICENSE "${root}/LICENSE"
|
install -m 0644 LICENSE "${root}/LICENSE"
|
||||||
|
|
||||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||||
done
|
|
||||||
done
|
done
|
||||||
|
|
||||||
sed \
|
sed \
|
||||||
@@ -362,8 +376,24 @@ jobs:
|
|||||||
chmod +x release-assets/install.sh
|
chmod +x release-assets/install.sh
|
||||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||||
|
|
||||||
|
- name: Attest release package provenance
|
||||||
|
id: attest-release
|
||||||
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-path: |
|
||||||
|
release-assets/*.tar.gz
|
||||||
|
release-assets/install.sh
|
||||||
|
release-assets/SHA256SUMS
|
||||||
|
|
||||||
|
- name: Bundle release package provenance
|
||||||
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
|
env:
|
||||||
|
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||||
|
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||||
|
|
||||||
- name: Upload release package artifact
|
- name: Upload release package artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: release-assets
|
name: release-assets
|
||||||
path: release-assets/*
|
path: release-assets/*
|
||||||
@@ -375,15 +405,18 @@ jobs:
|
|||||||
needs: [preflight, docker, package, vscodex]
|
needs: [preflight, docker, package, vscodex]
|
||||||
if: needs.preflight.outputs.publish == 'true'
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download release package artifact
|
- name: Download release package artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: release-assets
|
name: release-assets
|
||||||
path: release-assets
|
path: release-assets
|
||||||
|
|
||||||
- name: Download VSIX artifact
|
- name: Download VSIX artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-vscodex-vsix
|
name: aether-vscodex-vsix
|
||||||
path: release-assets
|
path: release-assets
|
||||||
@@ -410,13 +443,14 @@ jobs:
|
|||||||
done <<< "${draft_ids}"
|
done <<< "${draft_ids}"
|
||||||
|
|
||||||
- name: Publish GitHub Release assets
|
- name: Publish GitHub Release assets
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||||
with:
|
with:
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||||
files: |
|
files: |
|
||||||
release-assets/*.tar.gz
|
release-assets/*.tar.gz
|
||||||
|
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||||
release-assets/SHA256SUMS
|
release-assets/SHA256SUMS
|
||||||
release-assets/install.sh
|
release-assets/install.sh
|
||||||
release-assets/*.vsix
|
release-assets/*.vsix
|
||||||
|
|||||||
+315
-221
@@ -2,6 +2,12 @@ name: Rust CI
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_call:
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
full_scope:
|
||||||
|
description: "Run all Rust and shell scopes, used by Nightly"
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
@@ -9,18 +15,58 @@ on:
|
|||||||
paths:
|
paths:
|
||||||
- "Cargo.toml"
|
- "Cargo.toml"
|
||||||
- "Cargo.lock"
|
- "Cargo.lock"
|
||||||
|
- "rust-toolchain.toml"
|
||||||
|
- ".cargo/**"
|
||||||
- "crates/**"
|
- "crates/**"
|
||||||
- "apps/**"
|
- "apps/**"
|
||||||
- ".github/workflows/rust-ci.yml"
|
- "*.sql"
|
||||||
- ".github/workflows/nightly.yml"
|
- "install.sh"
|
||||||
|
- "deploy.sh"
|
||||||
|
- "update.sh"
|
||||||
|
- "generate_keys.sh"
|
||||||
|
- ".env.example"
|
||||||
|
- "README.md"
|
||||||
|
- "Dockerfile.app"
|
||||||
|
- "docker-compose.yml"
|
||||||
|
- "docker-compose.single-node.yml"
|
||||||
|
- "docker-compose.local.yml"
|
||||||
|
- "docker-compose.release-local.yml"
|
||||||
|
- "tests/compose_database_config_test.py"
|
||||||
|
- "tests/install_*_test.sh"
|
||||||
|
- "tests/deploy_*_test.sh"
|
||||||
|
- "tests/update_*_test.sh"
|
||||||
|
- "tests/release_supply_chain_test.sh"
|
||||||
|
- "tests/tunnel_installer_config_security_test.sh"
|
||||||
|
- ".github/workflows/*.yml"
|
||||||
|
- ".github/workflows/*.yaml"
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- "Cargo.toml"
|
- "Cargo.toml"
|
||||||
- "Cargo.lock"
|
- "Cargo.lock"
|
||||||
|
- "rust-toolchain.toml"
|
||||||
|
- ".cargo/**"
|
||||||
- "crates/**"
|
- "crates/**"
|
||||||
- "apps/**"
|
- "apps/**"
|
||||||
- ".github/workflows/rust-ci.yml"
|
- "*.sql"
|
||||||
- ".github/workflows/nightly.yml"
|
- "install.sh"
|
||||||
|
- "deploy.sh"
|
||||||
|
- "update.sh"
|
||||||
|
- "generate_keys.sh"
|
||||||
|
- ".env.example"
|
||||||
|
- "README.md"
|
||||||
|
- "Dockerfile.app"
|
||||||
|
- "docker-compose.yml"
|
||||||
|
- "docker-compose.single-node.yml"
|
||||||
|
- "docker-compose.local.yml"
|
||||||
|
- "docker-compose.release-local.yml"
|
||||||
|
- "tests/compose_database_config_test.py"
|
||||||
|
- "tests/install_*_test.sh"
|
||||||
|
- "tests/deploy_*_test.sh"
|
||||||
|
- "tests/update_*_test.sh"
|
||||||
|
- "tests/release_supply_chain_test.sh"
|
||||||
|
- "tests/tunnel_installer_config_security_test.sh"
|
||||||
|
- ".github/workflows/*.yml"
|
||||||
|
- ".github/workflows/*.yaml"
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: rust-ci-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
group: rust-ci-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
@@ -36,14 +82,106 @@ env:
|
|||||||
CARGO_TERM_COLOR: always
|
CARGO_TERM_COLOR: always
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
fmt:
|
changes:
|
||||||
name: Format
|
name: Detect Rust CI scope
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
rust: ${{ steps.scope.outputs.rust }}
|
||||||
|
shell: ${{ steps.scope.outputs.shell }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Classify changed paths
|
||||||
|
id: scope
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
RUST_CI_FULL_SCOPE: ${{ inputs.full_scope || false }}
|
||||||
|
run: |
|
||||||
|
# 任何命令失败都必须让本 job 失败,否则 git fetch/diff 出错后仍会写出
|
||||||
|
# rust=false/shell=false,下游会误判为“无需测试”而假绿放行。
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Nightly 通过 workflow_call 显式传入 full_scope;普通 push/PR 只按源码和构建
|
||||||
|
# 指纹触发 Rust jobs,安装脚本、Compose、README 等由 shell scope 覆盖。
|
||||||
|
if [ "$RUST_CI_FULL_SCOPE" = "true" ]; then
|
||||||
|
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] \
|
||||||
|
&& [ -n "${GITHUB_BASE_REF:-}" ] \
|
||||||
|
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||||
|
git fetch --no-tags origin "$GITHUB_BASE_REF" --depth=1
|
||||||
|
changed_paths=$(git diff --name-only "origin/$GITHUB_BASE_REF...$GITHUB_SHA")
|
||||||
|
elif [ "$GITHUB_EVENT_NAME" = "push" ] \
|
||||||
|
&& [ -n "${GITHUB_EVENT_BEFORE:-}" ] \
|
||||||
|
&& [ "$GITHUB_EVENT_BEFORE" != "0000000000000000000000000000000000000000" ] \
|
||||||
|
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||||
|
changed_paths=$(git diff --name-only "$GITHUB_EVENT_BEFORE" "$GITHUB_SHA")
|
||||||
|
else
|
||||||
|
changed_paths=$(git ls-files)
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 防御性兜底:diff 结果为空(异常事件或比较失败)时按全量运行,
|
||||||
|
# 宁可多跑也不能漏测。
|
||||||
|
if [ -z "$changed_paths" ]; then
|
||||||
|
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
rust=false
|
||||||
|
shell=false
|
||||||
|
while IFS= read -r path; do
|
||||||
|
case "$path" in
|
||||||
|
Cargo.toml|Cargo.lock|rust-toolchain.toml|.cargo/*|*.rs|*/Cargo.toml|*/build.rs|*.sql|.github/workflows/*.yml|.github/workflows/*.yaml)
|
||||||
|
rust=true
|
||||||
|
;;
|
||||||
|
*.sh|*.py|README.md|*/README.md|.env.example|Dockerfile*|docker-compose*.yml|docker-compose*.yaml)
|
||||||
|
shell=true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done <<< "$changed_paths"
|
||||||
|
|
||||||
|
echo "rust=$rust" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "shell=$shell" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
shell_security:
|
||||||
|
name: Shell security fixtures
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.shell == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
|
- name: Run installer and supply-chain fixtures
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python3 tests/compose_database_config_test.py
|
||||||
|
bash tests/deploy_state_safety_test.sh
|
||||||
|
bash tests/install_archive_safety_test.sh
|
||||||
|
bash tests/install_container_runtime_security_test.sh
|
||||||
|
bash tests/install_current_release_link_test.sh
|
||||||
|
bash tests/install_local_bundle_safety_test.sh
|
||||||
|
bash tests/install_privileged_write_safety_test.sh
|
||||||
|
bash tests/install_source_trust_test.sh
|
||||||
|
bash tests/release_supply_chain_test.sh
|
||||||
|
bash tests/update_compose_safety_test.sh
|
||||||
|
bash tests/tunnel_installer_config_security_test.sh
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
name: Format
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: rustfmt
|
components: rustfmt
|
||||||
@@ -53,24 +191,28 @@ jobs:
|
|||||||
|
|
||||||
clippy_gateway:
|
clippy_gateway:
|
||||||
name: Clippy (Gateway)
|
name: Clippy (Gateway)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
# Gateway lint 与 Gateway 测试都可能触发 mold/大型链接依赖,单独隔离缓存
|
||||||
|
# 指纹,避免不同 job 的构建产物互相驱逐或复用错误的链接参数。
|
||||||
|
shared-key: rust-ci-gateway-clippy-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -87,24 +229,26 @@ jobs:
|
|||||||
|
|
||||||
clippy_data:
|
clippy_data:
|
||||||
name: Clippy (Data)
|
name: Clippy (Data)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -121,24 +265,26 @@ jobs:
|
|||||||
|
|
||||||
clippy_rest:
|
clippy_rest:
|
||||||
name: Clippy (Workspace Rest)
|
name: Clippy (Workspace Rest)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -157,6 +303,7 @@ jobs:
|
|||||||
name: Clippy
|
name: Clippy
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
|
- changes
|
||||||
- clippy_gateway
|
- clippy_gateway
|
||||||
- clippy_data
|
- clippy_data
|
||||||
- clippy_rest
|
- clippy_rest
|
||||||
@@ -164,6 +311,10 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Verify clippy jobs
|
- name: Verify clippy jobs
|
||||||
run: |
|
run: |
|
||||||
|
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||||
|
echo "Rust scope unchanged; clippy jobs skipped"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
if [ "${{ needs.clippy_gateway.result }}" != "success" ] || \
|
if [ "${{ needs.clippy_gateway.result }}" != "success" ] || \
|
||||||
[ "${{ needs.clippy_data.result }}" != "success" ] || \
|
[ "${{ needs.clippy_data.result }}" != "success" ] || \
|
||||||
[ "${{ needs.clippy_rest.result }}" != "success" ]; then
|
[ "${{ needs.clippy_rest.result }}" != "success" ]; then
|
||||||
@@ -173,82 +324,100 @@ jobs:
|
|||||||
|
|
||||||
test_gateway:
|
test_gateway:
|
||||||
name: Test (Gateway)
|
name: Test (Gateway)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# 构建指纹提到 job 级:mold RUSTFLAGS / 栈 / sccache 对 lib、bins、integration 三步保持一致,
|
||||||
|
# 避免 step 级 env 漂移导致同 job 内 rustc 指纹不一致。
|
||||||
|
env:
|
||||||
|
RUSTC_WRAPPER: sccache
|
||||||
|
SCCACHE_GHA_ENABLED: "true"
|
||||||
|
RUST_MIN_STACK: "16777216"
|
||||||
|
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
# 与 rust-toolchain.toml、fmt/clippy 钉在同一版本,避免浮动 stable 换指纹导致全量重编
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
# mold RUSTFLAGS 只在本 job 生效:独立 cache key,避免与无 mold 的 job 互相污染指纹
|
||||||
|
shared-key: rust-ci-gateway-test-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Setup mold
|
- name: Setup mold
|
||||||
uses: rui314/setup-mold@v1
|
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test lib
|
- name: Test lib
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
RUST_MIN_STACK: "16777216"
|
|
||||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
|
||||||
run: cargo nextest run -p aether-gateway --lib
|
run: cargo nextest run -p aether-gateway --lib
|
||||||
|
|
||||||
- name: Test bins
|
- name: Test bins
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
RUST_MIN_STACK: "16777216"
|
|
||||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
|
||||||
run: cargo nextest run -p aether-gateway --bins
|
run: cargo nextest run -p aether-gateway --bins
|
||||||
|
|
||||||
|
# 只运行独立 integration targets;显式列出目标,避免 --tests 再次执行 lib/bin 测试。
|
||||||
|
- name: Test integration targets
|
||||||
|
run: >-
|
||||||
|
cargo nextest run -p aether-gateway
|
||||||
|
--test admin_unsigned_identity_headers
|
||||||
|
--test architecture_guard
|
||||||
|
|
||||||
- name: Show sccache stats
|
- name: Show sccache stats
|
||||||
if: always()
|
if: always()
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: sccache --show-stats
|
run: sccache --show-stats
|
||||||
|
|
||||||
test_data:
|
test_data:
|
||||||
name: Test (Data)
|
name: Test (Data)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test
|
- name: Test
|
||||||
env:
|
env:
|
||||||
RUSTC_WRAPPER: sccache
|
RUSTC_WRAPPER: sccache
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
SCCACHE_GHA_ENABLED: "true"
|
||||||
|
AETHER_REQUIRE_LOCAL_POSTGRES_TESTS: "true"
|
||||||
run: cargo nextest run -p aether-data
|
run: cargo nextest run -p aether-data
|
||||||
|
|
||||||
- name: Show sccache stats
|
- name: Show sccache stats
|
||||||
@@ -260,29 +429,31 @@ jobs:
|
|||||||
|
|
||||||
check_data_features:
|
check_data_features:
|
||||||
name: Check (Data Feature - ${{ matrix.feature }})
|
name: Check (Data Feature - ${{ matrix.feature }})
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
feature:
|
feature:
|
||||||
- postgres
|
- postgres
|
||||||
- mysql
|
|
||||||
- sqlite
|
|
||||||
- all-drivers
|
- all-drivers
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Check selected data driver
|
- name: Check selected data driver
|
||||||
env:
|
env:
|
||||||
@@ -299,27 +470,31 @@ jobs:
|
|||||||
|
|
||||||
test_rest:
|
test_rest:
|
||||||
name: Test (Workspace Rest)
|
name: Test (Workspace Rest)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
- name: Test
|
- name: Test
|
||||||
env:
|
env:
|
||||||
@@ -336,31 +511,33 @@ jobs:
|
|||||||
|
|
||||||
test_data_adapters:
|
test_data_adapters:
|
||||||
name: Test (Data Adapter - ${{ matrix.package }})
|
name: Test (Data Adapter - ${{ matrix.package }})
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
package:
|
package:
|
||||||
- aether-data-postgres
|
- aether-data-postgres
|
||||||
- aether-data-mysql
|
|
||||||
- aether-data-sqlite
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
- name: Test adapter
|
- name: Test adapter
|
||||||
env:
|
env:
|
||||||
@@ -377,21 +554,28 @@ jobs:
|
|||||||
|
|
||||||
check_integration_scenarios:
|
check_integration_scenarios:
|
||||||
name: Test (Integration Scenarios)
|
name: Test (Integration Scenarios)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test scenario binaries and end-to-end suites
|
- name: Test scenario binaries and end-to-end suites
|
||||||
env:
|
env:
|
||||||
@@ -410,6 +594,7 @@ jobs:
|
|||||||
name: Test
|
name: Test
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
|
- changes
|
||||||
- test_gateway
|
- test_gateway
|
||||||
- test_data
|
- test_data
|
||||||
- check_data_features
|
- check_data_features
|
||||||
@@ -420,6 +605,10 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Verify test jobs
|
- name: Verify test jobs
|
||||||
run: |
|
run: |
|
||||||
|
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||||
|
echo "Rust scope unchanged; test jobs skipped"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
if [ "${{ needs.test_gateway.result }}" != "success" ] || \
|
if [ "${{ needs.test_gateway.result }}" != "success" ] || \
|
||||||
[ "${{ needs.test_data.result }}" != "success" ] || \
|
[ "${{ needs.test_data.result }}" != "success" ] || \
|
||||||
[ "${{ needs.check_data_features.result }}" != "success" ] || \
|
[ "${{ needs.check_data_features.result }}" != "success" ] || \
|
||||||
@@ -430,42 +619,10 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
data_db_smoke_sqlite:
|
|
||||||
name: Data DB Smoke (SQLite)
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
|
||||||
run: rustup show active-toolchain
|
|
||||||
|
|
||||||
- name: Rust cache
|
|
||||||
uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
|
||||||
workspaces: . -> target
|
|
||||||
|
|
||||||
- name: Setup sccache
|
|
||||||
uses: mozilla-actions/[email protected]
|
|
||||||
|
|
||||||
- name: Run SQLite data smoke tests
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: cargo test -p aether-data --all-features sqlite --lib
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
|
||||||
if: always()
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: sccache --show-stats
|
|
||||||
|
|
||||||
data_db_smoke_postgres:
|
data_db_smoke_postgres:
|
||||||
name: Data DB Smoke (Postgres)
|
name: Data DB Smoke (Postgres)
|
||||||
|
needs: changes
|
||||||
|
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
services:
|
services:
|
||||||
postgres:
|
postgres:
|
||||||
@@ -482,22 +639,24 @@ jobs:
|
|||||||
--health-timeout=5s
|
--health-timeout=5s
|
||||||
--health-retries=20
|
--health-retries=20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
with:
|
||||||
|
toolchain: 1.95.0
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Add PostgreSQL server binaries to PATH
|
- name: Add PostgreSQL server binaries to PATH
|
||||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||||
@@ -516,6 +675,20 @@ jobs:
|
|||||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||||
run: cargo test -p aether-data --all-features postgres_provider_upstream_metadata_migration_preserves_json_when_url_is_set --lib -- --nocapture
|
run: cargo test -p aether-data --all-features postgres_provider_upstream_metadata_migration_preserves_json_when_url_is_set --lib -- --nocapture
|
||||||
|
|
||||||
|
- name: Run Postgres payment callback regression tests
|
||||||
|
env:
|
||||||
|
RUSTC_WRAPPER: sccache
|
||||||
|
SCCACHE_GHA_ENABLED: "true"
|
||||||
|
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||||
|
run: cargo test -p aether-data-postgres live_payment_callback --lib -- --ignored --nocapture
|
||||||
|
|
||||||
|
- name: Run Postgres batch wallet deduction regression
|
||||||
|
env:
|
||||||
|
RUSTC_WRAPPER: sccache
|
||||||
|
SCCACHE_GHA_ENABLED: "true"
|
||||||
|
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||||
|
run: cargo test -p aether-data-postgres live_bulk_wallet_adjustment_persists_actual_delta_and_skips_zero_ledger --lib -- --ignored --nocapture
|
||||||
|
|
||||||
- name: Run Postgres API key lifecycle tests
|
- name: Run Postgres API key lifecycle tests
|
||||||
env:
|
env:
|
||||||
RUSTC_WRAPPER: sccache
|
RUSTC_WRAPPER: sccache
|
||||||
@@ -535,112 +708,6 @@ jobs:
|
|||||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||||
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||||
|
|
||||||
- name: Run SQLite-to-Postgres import smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features sqlite_core_export_reads_migrated_database_rows --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
|
||||||
if: always()
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: sccache --show-stats
|
|
||||||
|
|
||||||
data_db_smoke_mysql:
|
|
||||||
name: Data DB Smoke (MySQL)
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
services:
|
|
||||||
mysql:
|
|
||||||
image: mysql:8.0
|
|
||||||
env:
|
|
||||||
MYSQL_DATABASE: aether_test
|
|
||||||
MYSQL_USER: aether
|
|
||||||
MYSQL_PASSWORD: aether
|
|
||||||
MYSQL_ROOT_PASSWORD: aether_root
|
|
||||||
ports:
|
|
||||||
- 3306:3306
|
|
||||||
options: >-
|
|
||||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uaether -paether --silent"
|
|
||||||
--health-interval=5s
|
|
||||||
--health-timeout=5s
|
|
||||||
--health-retries=20
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
|
||||||
run: rustup show active-toolchain
|
|
||||||
|
|
||||||
- name: Rust cache
|
|
||||||
uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
|
||||||
workspaces: . -> target
|
|
||||||
|
|
||||||
- name: Setup sccache
|
|
||||||
uses: mozilla-actions/[email protected]
|
|
||||||
|
|
||||||
- name: Run MySQL migration smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_migrations_create_core_config_tables_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL usage write smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_usage_write_repository_upserts_and_flushes_counters_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL usage read smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_usage_read_repository_reads_usage_contract_views_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL provider catalog smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_provider_catalog_repository_round_trips_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL core export smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL wallet read smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_wallet_read_repository_reads_wallet_contract_views --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL wallet daily usage aggregation smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL stats aggregation smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
- name: Show sccache stats
|
||||||
if: always()
|
if: always()
|
||||||
env:
|
env:
|
||||||
@@ -652,16 +719,21 @@ jobs:
|
|||||||
name: Data DB Smoke
|
name: Data DB Smoke
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
- data_db_smoke_sqlite
|
- changes
|
||||||
- data_db_smoke_postgres
|
- data_db_smoke_postgres
|
||||||
- data_db_smoke_mysql
|
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
steps:
|
steps:
|
||||||
- name: Verify database smoke jobs
|
- name: Verify database smoke jobs
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ needs.data_db_smoke_sqlite.result }}" != "success" ] || \
|
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||||
[ "${{ needs.data_db_smoke_postgres.result }}" != "success" ] || \
|
echo "Scope detection failed"
|
||||||
[ "${{ needs.data_db_smoke_mysql.result }}" != "success" ]; then
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||||
|
echo "Rust scope unchanged; database smoke jobs skipped"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||||
echo "Data DB smoke failed"
|
echo "Data DB smoke failed"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -670,18 +742,40 @@ jobs:
|
|||||||
name: check
|
name: check
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
|
- changes
|
||||||
- fmt
|
- fmt
|
||||||
- clippy
|
- clippy
|
||||||
- test
|
- test
|
||||||
- data_db_smoke
|
- data_db_smoke
|
||||||
|
- shell_security
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
steps:
|
steps:
|
||||||
- name: Verify required jobs
|
- name: Verify required jobs
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
# changes 失败或未产出 scope 时不允许直接放行,避免假绿。
|
||||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||||
[ "${{ needs.test.result }}" != "success" ] || \
|
echo "Scope detection failed"
|
||||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
exit 1
|
||||||
|
fi
|
||||||
|
rust="${{ needs.changes.outputs.rust }}"
|
||||||
|
shell="${{ needs.changes.outputs.shell }}"
|
||||||
|
|
||||||
|
if [ "$rust" != "true" ] && [ "$shell" != "true" ]; then
|
||||||
|
echo "No Rust or shell scope changed"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$rust" = "true" ] && {
|
||||||
|
[ "${{ needs.fmt.result }}" != "success" ] ||
|
||||||
|
[ "${{ needs.clippy.result }}" != "success" ] ||
|
||||||
|
[ "${{ needs.test.result }}" != "success" ] ||
|
||||||
|
[ "${{ needs.data_db_smoke.result }}" != "success" ];
|
||||||
|
}; then
|
||||||
|
echo "Rust CI failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$shell" = "true" ] && [ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||||
echo "Rust CI failed"
|
echo "Rust CI failed"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -13,6 +13,10 @@
|
|||||||
.plans
|
.plans
|
||||||
.playwright-mcp/
|
.playwright-mcp/
|
||||||
|
|
||||||
|
docs/architecture
|
||||||
|
!docs/architecture/architecture-dark.svg
|
||||||
|
!docs/architecture/architecture-light.svg
|
||||||
|
|
||||||
### Python ###
|
### Python ###
|
||||||
*.db
|
*.db
|
||||||
*.db-*
|
*.db-*
|
||||||
|
|||||||
Generated
+624
-361
File diff suppressed because it is too large
Load Diff
+4
-5
@@ -11,8 +11,6 @@ members = [
|
|||||||
"crates/aether-routing-core",
|
"crates/aether-routing-core",
|
||||||
"crates/aether-data/contracts",
|
"crates/aether-data/contracts",
|
||||||
"crates/aether-data/adapters/postgres",
|
"crates/aether-data/adapters/postgres",
|
||||||
"crates/aether-data/adapters/mysql",
|
|
||||||
"crates/aether-data/adapters/sqlite",
|
|
||||||
"crates/aether-data/query",
|
"crates/aether-data/query",
|
||||||
"crates/aether-data/schema",
|
"crates/aether-data/schema",
|
||||||
"crates/aether-dispatch-core",
|
"crates/aether-dispatch-core",
|
||||||
@@ -66,8 +64,6 @@ aether-provider-pool = { path = "crates/aether-provider/pool" }
|
|||||||
aether-routing-core = { path = "crates/aether-routing-core" }
|
aether-routing-core = { path = "crates/aether-routing-core" }
|
||||||
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
||||||
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
||||||
aether-data-mysql = { path = "crates/aether-data/adapters/mysql" }
|
|
||||||
aether-data-sqlite = { path = "crates/aether-data/adapters/sqlite" }
|
|
||||||
aether-data-query = { path = "crates/aether-data/query" }
|
aether-data-query = { path = "crates/aether-data/query" }
|
||||||
aether-data-schema = { path = "crates/aether-data/schema" }
|
aether-data-schema = { path = "crates/aether-data/schema" }
|
||||||
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
||||||
@@ -96,11 +92,13 @@ aether-usage-core = { path = "crates/aether-usage/core" }
|
|||||||
aether-usage-runtime = { path = "crates/aether-usage/runtime" }
|
aether-usage-runtime = { path = "crates/aether-usage/runtime" }
|
||||||
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
|
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
|
||||||
aether-gateway = { path = "apps/aether-gateway" }
|
aether-gateway = { path = "apps/aether-gateway" }
|
||||||
|
aether-tunnel = { path = "apps/aether-tunnel" }
|
||||||
aether-http = { path = "crates/aether-http" }
|
aether-http = { path = "crates/aether-http" }
|
||||||
aether-runtime = { path = "crates/aether-runtime/base" }
|
aether-runtime = { path = "crates/aether-runtime/base" }
|
||||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||||
aes = "0.8"
|
aes = "0.8"
|
||||||
aes-gcm = "0.10"
|
aes-gcm = "0.10"
|
||||||
|
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||||
async-stream = "0.3"
|
async-stream = "0.3"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
axum = "0.8"
|
axum = "0.8"
|
||||||
@@ -117,8 +115,9 @@ flate2 = "1"
|
|||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
hmac = "0.12"
|
hmac = "0.12"
|
||||||
http = "1"
|
http = "1"
|
||||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||||
|
percent-encoding = "2"
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||||
regex = "1"
|
regex = "1"
|
||||||
|
|||||||
+9
-4
@@ -10,20 +10,24 @@
|
|||||||
|
|
||||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||||
# distroless has no shell, so we use busybox to set up the symlink.
|
# distroless has no shell, so we use busybox to set up the symlink.
|
||||||
FROM busybox:1.37-musl AS layout
|
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||||
|
|
||||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
|
||||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||||
|
|
||||||
|
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||||
|
# identity can traverse and read every packaged asset.
|
||||||
|
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||||
|
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||||
|
|
||||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||||
|
|
||||||
# --- final stage: distroless runtime ---
|
# --- final stage: distroless runtime ---
|
||||||
FROM gcr.io/distroless/static-debian12
|
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||||
|
|
||||||
COPY --from=layout /opt/aether /opt/aether
|
COPY --from=layout /opt/aether /opt/aether
|
||||||
|
|
||||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
|||||||
|
|
||||||
ENV RUST_LOG=aether_gateway=info \
|
ENV RUST_LOG=aether_gateway=info \
|
||||||
APP_PORT=8084 \
|
APP_PORT=8084 \
|
||||||
|
HOME=/tmp/aether-home \
|
||||||
AETHER_UPDATE_STRATEGY=docker \
|
AETHER_UPDATE_STRATEGY=docker \
|
||||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||||
|
|
||||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
USER root
|
USER 0:0
|
||||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -157,4 +157,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
|
USER 0:0
|
||||||
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -156,4 +156,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
|
USER 0:0
|
||||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -36,15 +36,6 @@ lowercase() {
|
|||||||
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
||||||
}
|
}
|
||||||
|
|
||||||
dev_uses_sqlite_database() {
|
|
||||||
local driver
|
|
||||||
local url
|
|
||||||
driver="$$(lowercase "$${AETHER_DATABASE_DRIVER:-}")"
|
|
||||||
url="$${AETHER_DATABASE_URL:-$${DATABASE_URL:-}}"
|
|
||||||
|
|
||||||
[[ "$${driver}" == "sqlite" || "$${url}" == sqlite:* ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
dev_uses_postgres_database() {
|
dev_uses_postgres_database() {
|
||||||
local driver
|
local driver
|
||||||
local url
|
local url
|
||||||
@@ -68,9 +59,6 @@ dev_uses_redis_runtime() {
|
|||||||
if [[ "$${backend}" == "redis" ]]; then
|
if [[ "$${backend}" == "redis" ]]; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
if dev_uses_sqlite_database; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
@@ -356,6 +344,9 @@ if ! ensure_dev_infra; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "=> 编译 aether-gateway..."
|
||||||
|
cargo build -p aether-gateway --bin aether-gateway
|
||||||
|
|
||||||
GATEWAY_PID=""
|
GATEWAY_PID=""
|
||||||
GATEWAY_LOG_DIR=""
|
GATEWAY_LOG_DIR=""
|
||||||
GATEWAY_LOG_FILE=""
|
GATEWAY_LOG_FILE=""
|
||||||
@@ -364,8 +355,8 @@ create_gateway_log_file
|
|||||||
|
|
||||||
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
||||||
echo "=> 日志过滤: $${RUST_LOG}"
|
echo "=> 日志过滤: $${RUST_LOG}"
|
||||||
echo "=> 执行命令: cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT}"
|
echo "=> 执行命令: target/debug/aether-gateway --app-port $${APP_PORT}"
|
||||||
cargo run -p aether-gateway --bin aether-gateway -- --app-port "$${APP_PORT}" > >(
|
target/debug/aether-gateway --app-port "$${APP_PORT}" > >(
|
||||||
tee -a "$${GATEWAY_LOG_FILE}"
|
tee -a "$${GATEWAY_LOG_FILE}"
|
||||||
) 2>&1 &
|
) 2>&1 &
|
||||||
GATEWAY_PID=$$!
|
GATEWAY_PID=$$!
|
||||||
@@ -456,7 +447,7 @@ if [ -f .env ]; then
|
|||||||
fi
|
fi
|
||||||
export APP_PORT="$${APP_PORT:-8084}"
|
export APP_PORT="$${APP_PORT:-8084}"
|
||||||
|
|
||||||
echo "=> 启动后端: RUST_LOG=$${DEV_RUST_LOG} cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT:-8084}"
|
echo "=> 启动后端: 先编译 aether-gateway,再运行 target/debug/aether-gateway --app-port $${APP_PORT:-8084}"
|
||||||
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
||||||
backend_pid=$$!
|
backend_pid=$$!
|
||||||
|
|
||||||
|
|||||||
@@ -44,86 +44,39 @@ cd Aether
|
|||||||
|
|
||||||
# 2. 配置环境变量
|
# 2. 配置环境变量
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||||
|
chmod 600 .env
|
||||||
|
# 生成 JWT / 加密 / Postgres / Redis 独立随机密钥,并填入 .env
|
||||||
./generate_keys.sh
|
./generate_keys.sh
|
||||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||||
|
|
||||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
# 3. Docker 部署 / 更新(PostgreSQL + Redis)
|
||||||
# Postgres + Redis (适用于企业或多人使用)
|
|
||||||
docker compose pull && docker compose up -d
|
docker compose pull && docker compose up -d
|
||||||
# Single Node (适用于个人用户或朋友分享)
|
|
||||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 一键更新
|
### 一键安装(PostgreSQL + Redis)
|
||||||
|
|
||||||
Docker Compose 部署后,可在部署目录直接执行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./update.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
`update.sh` 会拉取最新 `app` 镜像并重建 `app` 容器,Docker named volumes、`./data` 和 `./logs` 不会被删除。Single Node 部署也可显式指定:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./update.sh --mode single-node
|
|
||||||
```
|
|
||||||
|
|
||||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
|
||||||
|
|
||||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
|
||||||
|
|
||||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
|
||||||
|
|
||||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
|
||||||
|
|
||||||
如果是本地源码构建镜像的部署,继续使用:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./deploy.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
如果要在本机联调“管理后台在线更新”本身,可启动仓库内置的 release-layout 测试环境:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.release-local.yml up -d --build
|
|
||||||
```
|
|
||||||
|
|
||||||
这套环境会用当前源码构建一个本地测试镜像,但编译为 `release` 类型,并默认伪装成 `v0.7.0`,这样后台会按正式发布版逻辑开放“立即更新”。默认监听 `http://127.0.0.1:18085`,数据目录使用 `./data-release-local`;日志默认走 `docker logs`,不会影响你正在跑的源码构建容器。
|
|
||||||
|
|
||||||
如果这套容器在 `prepare-update` 时访问 GitHub 失败,而你本机是通过代理出网,请在 `.env` 里把 `AETHER_UPDATE_PROXY_URL` 写成宿主机地址,例如 `http://host.docker.internal:7890`;容器内的 `127.0.0.1` 指向容器自身,不是宿主机。
|
|
||||||
|
|
||||||
如果想重置这套联调环境(包括 `/opt/aether/current` 和已下载的历史版本),执行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.release-local.yml down -v
|
|
||||||
```
|
|
||||||
|
|
||||||
可选变量:
|
|
||||||
|
|
||||||
- `AETHER_RELEASE_LOCAL_VERSION`:本地联调镜像对外声明的当前版本,默认 `v0.7.0`
|
|
||||||
- `AETHER_RELEASE_LOCAL_PORT`:本地联调端口,默认 `18085`
|
|
||||||
- `LOCAL_RELEASE_APP_IMAGE`:本地联调镜像名,默认 `aether-app:release-local`
|
|
||||||
|
|
||||||
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/fawney19/Aether.git
|
git clone https://github.com/fawney19/Aether.git
|
||||||
cd Aether
|
cd Aether
|
||||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash
|
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose
|
||||||
```
|
```
|
||||||
|
|
||||||
|
正式版和 Nightly 自动构建仅提供 Linux `amd64` / `arm64` 二进制包,Docker 镜像同样支持这两种架构。macOS 用户可使用 Docker 或自行从源码构建;安装脚本保留对历史 macOS 制品的兼容。独立 Aether Tunnel 的多平台发行不受此调整影响。
|
||||||
|
|
||||||
|
原生 Linux systemd 安装需先准备 PostgreSQL,将连接串通过 `DATABASE_URL` 传给安装进程,并选择 `--mode single-node`;不再自动创建本地数据库文件。
|
||||||
|
|
||||||
### Nightly(每日 main 构建)
|
### Nightly(每日 main 构建)
|
||||||
|
|
||||||
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
||||||
|
|
||||||
安装最新 nightly(Linux systemd / macOS launchd + SQLite):
|
安装最新 nightly(PostgreSQL + Redis):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --channel nightly
|
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose --channel nightly
|
||||||
```
|
```
|
||||||
|
|
||||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制方式可重新执行上述安装命令升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制部署请沿用已有 PostgreSQL 环境配置,并使用 `--mode single-node --channel nightly` 重新运行安装脚本升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||||
|
|
||||||
## 本地开发
|
## 本地开发
|
||||||
|
|
||||||
@@ -136,7 +89,7 @@ make dev
|
|||||||
|
|
||||||
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
||||||
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
||||||
数据库 schema 和历史数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。排查或部署前预执行时可使用:
|
`make dev` 会先完成后端编译,再开始计算服务健康检查超时。数据库 schema 和必要的派生数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。升级不会主动重写或清除已有业务历史记录,新写入会直接遵循当前的数据持久化策略。排查或部署前预执行时可使用:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make db-status
|
make db-status
|
||||||
@@ -168,26 +121,57 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
|||||||
## 环境变量
|
## 环境变量
|
||||||
|
|
||||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||||
- `DATABASE_URL`:数据库连接串;SQLite 例如 `sqlite:///opt/aether/data/aether.db`,Postgres 例如 `postgresql://postgres:aether@postgres:5432/aether`
|
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||||
|
- `AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS` / `AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS`:普通数据库连接的单条 SQL / 锁等待期限,默认 `30000` / `3000` 毫秒,显式 `0` 关闭;不是整个事务总期限。schema 迁移使用独立超时配置,历史 backfill 使用独立连接放宽期限
|
||||||
|
- `AETHER_POSTGRES_MIGRATION_LOCK_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_CONCURRENT_TIMEOUT_MS`:schema 迁移的锁等待、每个事务及并发索引迁移期限,默认 `1000` / `10000` / `900000` 毫秒,不接受 `0`。超时会中止当前迁移,已提交的迁移保留;空库 schema 初始化也受事务期限约束
|
||||||
|
- `AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES`:usage 诊断正文共享预算,默认 `134217728`(128 MiB),按 JSON 堆内存估算,覆盖进入终态队列的 seed、Redis 解码后的事件、数据库写入 DTO 及其正文副本。额度不足或显式 `0` 时先保留计费事实,再舍弃诊断正文;已有清空或禁用状态保持不变,其余标记截断。预算随正文保留到释放,后台构建或压缩不会因调用方取消而提前归还额度。该额度不覆盖原始 Redis 批次、解码临时分配、序列化及压缩结果、协议观察缓冲或进程总内存;可通过 `usage_runtime_event_capture_memory_*` 指标观察
|
||||||
|
- `AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES`:新增 usage 队列消息的完整 JSON payload 上限,默认 `1048576`(1 MiB),按序列化后的 UTF-8 字节计算,显式 `0` 非法。超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义时拒绝入队,终态消息尝试受限数据库落库,失败则明确失败,不继续 Redis 重试。该限制不覆盖存量 Redis 消息、整个读取批次、DLQ 或进程总内存。`usage_runtime_queue_payload_*` 导出上限及进程级降级、拒绝编码尝试次数,包含入队和重试预校验,不代表唯一事件数;`usage_runtime_enqueue_retry_permanent_failure_total` 记录永久输入错误导致的重试拒绝或终止
|
||||||
|
- `AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES` / `AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES`:usage worker 读取和重领共用的进程级逻辑 payload 预留,默认总额 `134217728`(128 MiB)、单批目标 `8388608`(8 MiB)。按当前 `QUEUE_PAYLOAD_MAX_BYTES` 推导实际 COUNT,默认最多读取 8 条,自动扩容使用实际 COUNT 判断批次是否读满。预留覆盖读取、整批处理和确认,额度不足等待;取消/失败释放。单批目标至少允许一条,当前 payload 上限大于总额时读取报配置错误。`0` 或非法值回退默认,过大值收敛到约 4 GiB 的有效总额。收到消息后按全部字段值长度缩减多余预留;历史消息、其他生产者使用更高上限或额外字段可能超出估算,仍继续原计费流程并记录 `usage_runtime_queue_read_oversized_*`。`usage_runtime_queue_read_*` 同时导出预留、等待与累计字段字节;该预留不是 RESP 解码、连接缓冲容量、字段结构、诊断 JSON、DLQ 或进程 RSS 的硬上限,旧公开 Vec 读取接口不携带处理阶段预留
|
||||||
|
- `AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES` / `AETHER_USAGE_DLQ_ENCODING_MAX_JOBS`:死信原文和 JSON 编码独立共享预留,默认 `67108864`(64 MiB)、最多 `4` 个后台编码及写入任务。根据原始字段、ID、错误字符串及 JSON 最坏 6 倍转义一次预留;预算占满或单条超总额时立即失败,worker 保留原消息等待重领,不截断账务原文。编码失败会继续处理同批其他消息,只确认成功项,批次末尾仍报告失败;存储转移失败则停止该批后续处理。取消编码等待不会提前归还仍在后台使用的额度。`0`/非法值回退默认,bytes 最大约 4 GiB,jobs 最大 128;超大存量消息可能需要调高总额后恢复。`usage_runtime_dlq_encoding_*` 导出额度、在途任务、拒绝和编码尝试次数;不包含字段结构、字符串额外容量、Redis 命令/连接副本或进程 RSS。内置 Redis/Memory worker 将死信追加、源 ACK 和删除作为一次原子转移,同一源 stream、消费组及 pending ID 的并发或重试只追加一次;Redis 要求 7+ 及 `EVAL/TYPE/XPENDING/XADD/XACK/XDEL` 权限,Cluster 两键须同 slot(当前默认键不自动迁移)。源和 DLQ 不能同名。源已不在 PEL 时不宣称已归档;外部 ACK/trim/delete 及多消费组仍有原来的删除语义。公开 `push_dead_letter` 仍为追加接口,未实现新原子 trait 方法的外部后端沿用追加后 ACK,仍可能重复归档
|
||||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
- `AETHER_GATEWAY_MAX_HTTP_CONNECTIONS`:二进制入口全部监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和 HTTP 升级后仍存活的 socket。未设置或 `0` 时使用请求上限与 WebSocket 上限之和;自动及显式值均最多 `65536`,已知 FD soft limit 时进一步限制为 `max(1, (FD - 256) / 2)`。接入后立即尝试取得额度,满额时关闭新连接,不创建 HTTP 处理任务、不等待额度,不返回 HTTP 状态码;取消、解析失败和连接释放归还,WebSocket 升级不会提前归还。HTTP/2 多流共用一个 TCP 许可,原请求和 WebSocket 准入仍独立有效。`gateway_http_connections_*` 导出配置上限、当前数、高水位、拒绝数及 accept 错误数。该限制不包含 kernel backlog、上游、Redis 或数据库连接,也不是整个进程 FD/内存硬上限。临时 accept 错误重试,资源类错误退避一秒后重试,避免单次错误停止监听
|
||||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`;压缩和未知长度上传按实际缓冲增长申请额度,解压时计入同时存活的输入和输出。额度不足返回 `503`;接近单请求上限的压缩上传需要为输入和解压输出预留额外预算
|
||||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`;显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
- `AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS`:上游流首包后的空闲超时,默认 `300000ms`;请求执行配置中的 `read_ms` 优先,显式 `0` 关闭对应超时。网关生成的 keepalive 不会重置计时
|
||||||
|
- `AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES`:进程内流式响应诊断捕获的共享字节预算,默认 `134217728`(128 MiB);包含 provider/client 捕获容量和扩容时的新旧分配。额度不足时仅截断审计副本,显式 `0` 关闭此类捕获;协议解析、客户端传输和计费观察继续执行。该预算不包含协议解析缓冲、终态编码及 usage 队列副本,不是进程总内存上限
|
||||||
|
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
|
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||||
|
- `AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS`:跨网关 owner relay 解析到私有/保留地址时的显式运维开关,默认关闭;仅当多网关 relay URL 是受控的内网 HTTPS 地址时设置为 `true`。它不改变普通 provider 请求的 DNS/代理策略,也不允许明文 HTTP 非 loopback relay
|
||||||
|
- `AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST`:更窄的 owner relay 私网例外,填写逗号分隔的精确主机名(例如 `gateway-a.internal,gateway-b.internal`,忽略大小写和末尾点);仅这些主机解析出的私有地址会被允许,并且请求仍使用解析后地址 pin。不要填写通配符或 `.internal` 这类后缀
|
||||||
|
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。配置 Redis 时使用 `redis`,否则使用 `memory`;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||||
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||||
|
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
- `DB_PASSWORD` / `REDIS_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||||
|
|
||||||
|
运行日志由独立后台线程写入 stdout 和文件,每个输出队列最多 4096 条、保留正文最多 8 MiB(包含正在写入的记录),单条最多 256 KiB。队列满、正文预算不足或单条超限时整条丢弃,不等待日志设备;`Both` 两个输出独立降级。`logging_stdout_*` 和 `logging_file_*` 指标记录丢弃和写入错误,网关指标沿用其命名空间前缀。正常退出时日志最多等待 2 秒排空;这不是请求优雅排空或整个进程退出期限。日志格式化仍在调用线程执行,日志预算不包含格式化临时内存,运行日志也不能作为可靠计费账本。
|
||||||
|
|
||||||
|
### S3 备份离线恢复
|
||||||
|
|
||||||
|
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||||
|
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||||
|
--input ./backup.json.zst.aes256gcm \
|
||||||
|
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||||
|
--output ./restored-backup.json
|
||||||
|
```
|
||||||
|
|
||||||
|
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||||
|
|
||||||
|
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ npm --prefix aether-vscodex/web run build
|
|||||||
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
||||||
共享连接目录前横向扩容 sidecar。
|
共享连接目录前横向扩容 sidecar。
|
||||||
|
|
||||||
登录 Aether 后打开“Codex 远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
登录 Aether 后打开“远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||||
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
||||||
VS Code SecretStorage,并同时保持本机控制台连接。
|
VS Code SecretStorage,并同时保持本机控制台连接。
|
||||||
|
|
||||||
|
|||||||
@@ -62,17 +62,18 @@ flate2.workspace = true
|
|||||||
futures-util.workspace = true
|
futures-util.workspace = true
|
||||||
hmac.workspace = true
|
hmac.workspace = true
|
||||||
http.workspace = true
|
http.workspace = true
|
||||||
|
http-body = "1"
|
||||||
http-body-util = "0.1"
|
http-body-util = "0.1"
|
||||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
md-5 = "0.10"
|
md-5 = "0.10"
|
||||||
object_store.workspace = true
|
object_store.workspace = true
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
percent-encoding.workspace = true
|
||||||
regex.workspace = true
|
regex.workspace = true
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
rsa = "0.9.10"
|
|
||||||
rustls.workspace = true
|
rustls.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
@@ -81,7 +82,7 @@ sha1 = "0.10"
|
|||||||
sha2 = { workspace = true, features = ["oid"] }
|
sha2 = { workspace = true, features = ["oid"] }
|
||||||
socket2.workspace = true
|
socket2.workspace = true
|
||||||
tar.workspace = true
|
tar.workspace = true
|
||||||
sqlx = { workspace = true, features = ["postgres", "mysql", "sqlite", "migrate"] }
|
sqlx = { workspace = true, features = ["postgres", "migrate"] }
|
||||||
sysinfo = "0.32"
|
sysinfo = "0.32"
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
@@ -102,5 +103,7 @@ tikv-jemallocator = { version = "0.6", optional = true }
|
|||||||
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
aether-testkit = { workspace = true, features = ["postgres"] }
|
||||||
aether-test-support.workspace = true
|
aether-test-support.workspace = true
|
||||||
|
aws-lc-rs.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ struct Args {
|
|||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||||
default_value = "/tmp/aether-execution-runtime.sock"
|
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||||
)]
|
)]
|
||||||
unix_socket: PathBuf,
|
unix_socket: PathBuf,
|
||||||
|
|
||||||
@@ -71,8 +71,13 @@ struct Args {
|
|||||||
distributed_request_command_timeout_ms: u64,
|
distributed_request_command_timeout_ms: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||||
|
run()
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||||
|
|
||||||
init_service_runtime(ServiceRuntimeConfig::new(
|
init_service_runtime(ServiceRuntimeConfig::new(
|
||||||
|
|||||||
@@ -88,8 +88,13 @@ struct Args {
|
|||||||
distributed_request_command_timeout_ms: u64,
|
distributed_request_command_timeout_ms: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||||
|
run()
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
init_service_runtime(ServiceRuntimeConfig::new(
|
init_service_runtime(ServiceRuntimeConfig::new(
|
||||||
"aether-tunnel-standalone",
|
"aether-tunnel-standalone",
|
||||||
"aether_gateway=info",
|
"aether_gateway=info",
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
pub(crate) use crate::handlers::admin::{
|
pub(crate) use crate::handlers::admin::{
|
||||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||||
reconcile_admin_fixed_provider_template_endpoints,
|
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||||
|
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||||
OAUTH_REQUEST_FAILED_PREFIX,
|
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::handlers::admin::{
|
use crate::handlers::admin::{
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
let body_bytes =
|
||||||
.decode(body_base64)
|
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
let Some(normalized_bytes) =
|
let Some(normalized_bytes) =
|
||||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||||
else {
|
else {
|
||||||
|
|||||||
@@ -69,9 +69,14 @@ pub(crate) use aether_ai_formats::api::{
|
|||||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||||
};
|
};
|
||||||
pub(crate) use aether_ai_formats::protocol::stream::CanonicalUsage as StreamingCanonicalUsage;
|
pub(crate) use aether_ai_formats::protocol::stream::CanonicalUsage as StreamingCanonicalUsage;
|
||||||
/// Codex client identity headers re-exported for out-of-crate probe binaries,
|
/// Codex client identity accessors re-exported for out-of-crate probe binaries,
|
||||||
/// which must reach `aether_ai_formats` through this seam.
|
/// which must reach the runtime profile through this seam.
|
||||||
pub use aether_ai_formats::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
pub use aether_ai_formats::{codex_client_originator, codex_client_user_agent};
|
||||||
|
/// Codex 动态客户端画像 API 只允许经此根缝进入 gateway,避免其它模块直接依赖 formats crate。
|
||||||
|
pub(crate) use aether_ai_formats::{
|
||||||
|
codex_client_profile, codex_client_version, set_codex_cli_version, set_codex_client_profile,
|
||||||
|
CodexClientProfile,
|
||||||
|
};
|
||||||
pub(crate) use aether_ai_formats::{CODEX_RESPONSES_LITE_HEADER, UPSTREAM_IS_STREAM_KEY};
|
pub(crate) use aether_ai_formats::{CODEX_RESPONSES_LITE_HEADER, UPSTREAM_IS_STREAM_KEY};
|
||||||
|
|
||||||
pub(crate) fn parse_direct_request_body(
|
pub(crate) fn parse_direct_request_body(
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use aether_provider_transport::CodexFingerprintConvergenceContext;
|
|
||||||
use http::{request::Parts, HeaderMap};
|
use http::{request::Parts, HeaderMap};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||||
use crate::client_session_affinity::codex_request_signals_from_request;
|
use crate::client_session_affinity::codex_request_signals_from_request;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<CodexFingerprintConvergenceContext>>);
|
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<ProviderOutboundRequestContext>>);
|
||||||
|
|
||||||
impl Default for CodexFingerprintContextSlot {
|
impl Default for CodexFingerprintContextSlot {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
@@ -18,11 +18,7 @@ impl Default for CodexFingerprintContextSlot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CodexFingerprintContextSlot {
|
impl CodexFingerprintContextSlot {
|
||||||
fn resolve(
|
fn resolve(&self, headers: &HeaderMap, body_json: &Value) -> ProviderOutboundRequestContext {
|
||||||
&self,
|
|
||||||
headers: &HeaderMap,
|
|
||||||
body_json: &Value,
|
|
||||||
) -> CodexFingerprintConvergenceContext {
|
|
||||||
self.0
|
self.0
|
||||||
.get_or_init(|| {
|
.get_or_init(|| {
|
||||||
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
||||||
@@ -34,10 +30,10 @@ impl CodexFingerprintContextSlot {
|
|||||||
pub(crate) fn resolve_codex_fingerprint_context(
|
pub(crate) fn resolve_codex_fingerprint_context(
|
||||||
parts: &Parts,
|
parts: &Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
if let Some(context) = parts
|
if let Some(context) = parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.cloned()
|
.cloned()
|
||||||
{
|
{
|
||||||
return context;
|
return context;
|
||||||
@@ -51,7 +47,7 @@ pub(crate) fn resolve_codex_fingerprint_context(
|
|||||||
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||||
if parts
|
if parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.is_none()
|
.is_none()
|
||||||
&& parts
|
&& parts
|
||||||
.extensions
|
.extensions
|
||||||
@@ -67,11 +63,11 @@ pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
|||||||
pub(crate) fn ensure_codex_fingerprint_context(
|
pub(crate) fn ensure_codex_fingerprint_context(
|
||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let context = resolve_codex_fingerprint_context(parts, body_json);
|
let context = resolve_codex_fingerprint_context(parts, body_json);
|
||||||
if parts
|
if parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.is_none()
|
.is_none()
|
||||||
{
|
{
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
@@ -84,7 +80,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
|||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
logical_turn_id: &str,
|
logical_turn_id: &str,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let context =
|
let context =
|
||||||
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
@@ -94,7 +90,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
|||||||
|
|
||||||
pub(crate) fn restore_codex_logical_turn_context(
|
pub(crate) fn restore_codex_logical_turn_context(
|
||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
context: &CodexFingerprintConvergenceContext,
|
context: &ProviderOutboundRequestContext,
|
||||||
) {
|
) {
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
parts.extensions.insert(context.clone());
|
parts.extensions.insert(context.clone());
|
||||||
@@ -104,10 +100,9 @@ fn build_codex_fingerprint_context(
|
|||||||
headers: &HeaderMap,
|
headers: &HeaderMap,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
logical_turn_id: String,
|
logical_turn_id: String,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
||||||
let mut context =
|
let mut context = ProviderOutboundRequestContext::new(logical_turn_id, current_unix_millis());
|
||||||
CodexFingerprintConvergenceContext::new(logical_turn_id, current_unix_millis());
|
|
||||||
|
|
||||||
if let Some(turn_id) = signals.turn_id {
|
if let Some(turn_id) = signals.turn_id {
|
||||||
context = context.with_original_turn_id(turn_id);
|
context = context.with_original_turn_id(turn_id);
|
||||||
@@ -160,14 +155,14 @@ mod tests {
|
|||||||
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
||||||
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
parts.extensions.get::<CodexFingerprintConvergenceContext>(),
|
parts.extensions.get::<ProviderOutboundRequestContext>(),
|
||||||
Some(&context)
|
Some(&context)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn restored_context_wins_over_retry_request_signals() {
|
fn restored_context_wins_over_retry_request_signals() {
|
||||||
let original = CodexFingerprintConvergenceContext::new("logical-turn", 1234)
|
let original = ProviderOutboundRequestContext::new("logical-turn", 1234)
|
||||||
.with_original_turn_id("original-turn")
|
.with_original_turn_id("original-turn")
|
||||||
.with_original_client_session_id("original-thread")
|
.with_original_client_session_id("original-thread")
|
||||||
.with_original_prompt_cache_key("original-cache");
|
.with_original_prompt_cache_key("original-cache");
|
||||||
|
|||||||
@@ -18,6 +18,12 @@ pub(crate) fn maybe_build_local_stream_rewriter<'a>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl LocalStreamRewriter<'_> {
|
impl LocalStreamRewriter<'_> {
|
||||||
|
pub(crate) fn into_owned(self) -> LocalStreamRewriter<'static> {
|
||||||
|
LocalStreamRewriter {
|
||||||
|
inner: self.inner.into_owned(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn push_chunk(&mut self, chunk: &[u8]) -> Result<Vec<u8>, GatewayError> {
|
pub(crate) fn push_chunk(&mut self, chunk: &[u8]) -> Result<Vec<u8>, GatewayError> {
|
||||||
self.inner.push_chunk(chunk).map_err(map_surface_error)
|
self.inner.push_chunk(chunk).map_err(map_surface_error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,10 +11,10 @@ use super::{
|
|||||||
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
||||||
maybe_build_local_core_sync_finalize_response,
|
maybe_build_local_core_sync_finalize_response,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::GatewayControlDecision;
|
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
convert_openai_chat_response_to_openai_responses,
|
convert_openai_chat_response_to_openai_responses,
|
||||||
convert_openai_responses_response_to_openai_chat,
|
convert_openai_responses_response_to_openai_chat, openai_responses_message_item_id,
|
||||||
|
GatewayControlDecision,
|
||||||
};
|
};
|
||||||
use crate::usage::GatewaySyncReportRequest;
|
use crate::usage::GatewaySyncReportRequest;
|
||||||
|
|
||||||
@@ -192,7 +192,7 @@ fn aggregates_openai_responses_stream_completed_event_to_final_response() {
|
|||||||
"output_text": "Hello",
|
"output_text": "Hello",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_123_msg",
|
"id": openai_responses_message_item_id("resp_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -843,7 +843,7 @@ fn converts_claude_cli_response_to_openai_responses_response() {
|
|||||||
"output_text": "Hello Claude CLI",
|
"output_text": "Hello Claude CLI",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "msg_cli_123_msg",
|
"id": openai_responses_message_item_id("msg_cli_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -907,7 +907,7 @@ fn converts_claude_cli_tool_use_to_openai_responses_function_call() {
|
|||||||
"output": [
|
"output": [
|
||||||
{
|
{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "msg_cli_tool_123_msg",
|
"id": openai_responses_message_item_id("msg_cli_tool_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -977,7 +977,7 @@ fn converts_gemini_cli_response_to_openai_responses_response() {
|
|||||||
"output_text": "Hello Gemini CLI",
|
"output_text": "Hello Gemini CLI",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -1046,7 +1046,7 @@ fn converts_gemini_cli_function_call_to_openai_responses_function_call() {
|
|||||||
"output": [
|
"output": [
|
||||||
{
|
{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_tool_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_tool_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -1252,7 +1252,7 @@ fn local_finalize_handles_openai_responses_openai_family_sync_response_even_when
|
|||||||
"model": "gpt-5",
|
"model": "gpt-5",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_family_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_family_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
|
|||||||
@@ -996,7 +996,7 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if page_is_exact_auth_api_key_concurrency_limited(&page) {
|
if page_is_exact_auth_api_key_concurrency_limited(&page) {
|
||||||
if self.wait_for_auth_api_key_concurrency_retry().await {
|
if self.wait_for_auth_api_key_concurrency_retry().await? {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
self.persist_final_auth_api_key_concurrency_skips(page.skipped_candidates)
|
self.persist_final_auth_api_key_concurrency_skips(page.skipped_candidates)
|
||||||
@@ -1087,20 +1087,23 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> bool {
|
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> Result<bool, GatewayError> {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let deadline = *self
|
let deadline = *self
|
||||||
.auth_api_key_concurrency_wait_deadline
|
.auth_api_key_concurrency_wait_deadline
|
||||||
.get_or_insert(now + AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
.get_or_insert(now + AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||||
if now >= deadline {
|
if !crate::scheduler::candidate::wait_for_auth_api_key_concurrency_retry(
|
||||||
return false;
|
self.state.app(),
|
||||||
|
Some(&self.auth_snapshot),
|
||||||
|
deadline,
|
||||||
|
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let sleep_duration =
|
|
||||||
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY.min(deadline.saturating_duration_since(now));
|
|
||||||
tokio::time::sleep(sleep_duration).await;
|
|
||||||
self.page_cursor.restart_scan();
|
self.page_cursor.restart_scan();
|
||||||
true
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn persist_final_auth_api_key_concurrency_skips(
|
async fn persist_final_auth_api_key_concurrency_skips(
|
||||||
@@ -2289,6 +2292,103 @@ mod tests {
|
|||||||
candidate
|
candidate
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn auth_concurrency_wait_paged_scan_retries_once_at_original_deadline() {
|
||||||
|
let now = current_unix_ms();
|
||||||
|
let active = serde_json::from_value(json!({
|
||||||
|
"id": "active-candidate",
|
||||||
|
"request_id": "active-request",
|
||||||
|
"api_key_id": "api-key-1",
|
||||||
|
"candidate_index": 0,
|
||||||
|
"retry_index": 0,
|
||||||
|
"status": "pending",
|
||||||
|
"is_cached": false,
|
||||||
|
"created_at_unix_ms": now,
|
||||||
|
"started_at_unix_ms": now
|
||||||
|
}))
|
||||||
|
.expect("active candidate should build");
|
||||||
|
let repository = Arc::new(InMemoryRequestCandidateRepository::seed([active]));
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_request_candidate_repository_for_tests(repository),
|
||||||
|
);
|
||||||
|
let mut auth_snapshot = sample_auth_snapshot();
|
||||||
|
auth_snapshot.api_key_concurrent_limit = Some(1);
|
||||||
|
let page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
|
PlannerAppState::new(&app),
|
||||||
|
&crate::system_features::ModelDirectivePolicySnapshot::default(),
|
||||||
|
"openai:chat",
|
||||||
|
"gpt-5",
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
&auth_snapshot,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModel,
|
||||||
|
true,
|
||||||
|
Some("trace-auth-wait"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut cursor = RequestedModelAttemptPageCursor {
|
||||||
|
state: PlannerAppState::new(&app),
|
||||||
|
trace_id: "trace-auth-wait".to_string(),
|
||||||
|
client_api_format: "openai:chat".to_string(),
|
||||||
|
requested_model: "gpt-5".to_string(),
|
||||||
|
auth_snapshot,
|
||||||
|
client_session_affinity: None,
|
||||||
|
required_capabilities: None,
|
||||||
|
routing_policy: None,
|
||||||
|
sticky_session_token: None,
|
||||||
|
request_auth_channel: None,
|
||||||
|
skipped_user_id: "user-1".to_string(),
|
||||||
|
skipped_api_key_id: "api-key-1".to_string(),
|
||||||
|
skipped_required_capabilities: None,
|
||||||
|
skipped_error_context: "test auth wait",
|
||||||
|
record_runtime_miss_diagnostic: false,
|
||||||
|
resolution_mode: LocalCandidateResolutionMode::Standard,
|
||||||
|
decorate_skipped_candidate: Arc::new(identity_skipped_candidate),
|
||||||
|
page_cursor,
|
||||||
|
pending_items: VecDeque::new(),
|
||||||
|
skipped_provider_ids: BTreeSet::new(),
|
||||||
|
skipped_endpoint_ids: BTreeSet::new(),
|
||||||
|
skipped_credential_ids: BTreeSet::new(),
|
||||||
|
candidate_count: 0,
|
||||||
|
next_candidate_index: 0,
|
||||||
|
remembered_affinity: false,
|
||||||
|
scheduler_cache_affinity_enabled: false,
|
||||||
|
auth_api_key_concurrency_wait_deadline: None,
|
||||||
|
deferred_error: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let started = Instant::now();
|
||||||
|
let mut scan_restarts = 0;
|
||||||
|
while cursor
|
||||||
|
.wait_for_auth_api_key_concurrency_retry()
|
||||||
|
.await
|
||||||
|
.expect("auth wait should succeed")
|
||||||
|
{
|
||||||
|
scan_restarts += 1;
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
scan_restarts, 1,
|
||||||
|
"blocked polls must not restart page scans"
|
||||||
|
);
|
||||||
|
assert!(started.elapsed() >= AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||||
|
let original_deadline = cursor.auth_api_key_concurrency_wait_deadline;
|
||||||
|
assert!(!cursor
|
||||||
|
.wait_for_auth_api_key_concurrency_retry()
|
||||||
|
.await
|
||||||
|
.expect("expired auth wait should succeed"));
|
||||||
|
assert_eq!(
|
||||||
|
cursor.auth_api_key_concurrency_wait_deadline,
|
||||||
|
original_deadline
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn pool_group_keys_are_not_persisted_as_available_before_attempt() {
|
async fn pool_group_keys_are_not_persisted_as_available_before_attempt() {
|
||||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||||
@@ -2325,16 +2425,11 @@ mod tests {
|
|||||||
assert_eq!(stored.len(), 1);
|
assert_eq!(stored.len(), 1);
|
||||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||||
assert_eq!(stored[0].candidate_index, 2);
|
assert_eq!(stored[0].candidate_index, 2);
|
||||||
assert_eq!(
|
assert!(stored[0]
|
||||||
stored[0]
|
.extra_data
|
||||||
.extra_data
|
.as_ref()
|
||||||
.as_ref()
|
.and_then(|value| value.get("dispatch_ref"))
|
||||||
.and_then(|value| value.get("dispatch_ref"))
|
.is_none());
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("normal-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2482,14 +2577,25 @@ mod tests {
|
|||||||
|
|
||||||
assert!(should_cache_resolved_candidate_page(&cursor));
|
assert!(should_cache_resolved_candidate_page(&cursor));
|
||||||
|
|
||||||
let fixed_order_app = AppState::new()
|
let fixed_order_app = AppState::new().expect("state should build");
|
||||||
.expect("state should build")
|
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||||
.with_data_state_for_tests(
|
billing_multiplier: 1.0,
|
||||||
GatewayDataState::disabled().with_system_config_values_for_tests([(
|
group_name: None,
|
||||||
"scheduling_mode".to_string(),
|
group_id: Some("routing-group-fixed-order".to_string()),
|
||||||
json!("fixed_order"),
|
group_version: Some(1),
|
||||||
)]),
|
selection_source: "test".to_string(),
|
||||||
);
|
requested_model: "gpt-5".to_string(),
|
||||||
|
resolved_model: "gpt-5".to_string(),
|
||||||
|
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||||
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
|
keep_priority_on_conversion: false,
|
||||||
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
|
ranking_overlay: Default::default(),
|
||||||
|
mutation_plan: Default::default(),
|
||||||
|
pool_policy_overrides: Default::default(),
|
||||||
|
matched_rules: Vec::new(),
|
||||||
|
};
|
||||||
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
PlannerAppState::new(&fixed_order_app),
|
PlannerAppState::new(&fixed_order_app),
|
||||||
&model_directive_policy,
|
&model_directive_policy,
|
||||||
@@ -2499,7 +2605,7 @@ mod tests {
|
|||||||
true,
|
true,
|
||||||
None,
|
None,
|
||||||
&auth_snapshot,
|
&auth_snapshot,
|
||||||
None,
|
Some(&fixed_order_policy),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
@@ -2517,7 +2623,7 @@ mod tests {
|
|||||||
auth_snapshot,
|
auth_snapshot,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
required_capabilities: None,
|
required_capabilities: None,
|
||||||
routing_policy: None,
|
routing_policy: Some(fixed_order_policy),
|
||||||
sticky_session_token: None,
|
sticky_session_token: None,
|
||||||
request_auth_channel: None,
|
request_auth_channel: None,
|
||||||
skipped_user_id: "user-1".to_string(),
|
skipped_user_id: "user-1".to_string(),
|
||||||
@@ -2615,16 +2721,11 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||||
assert_eq!(stored[1].candidate_index, 1);
|
assert_eq!(stored[1].candidate_index, 1);
|
||||||
assert_eq!(
|
assert!(stored[1]
|
||||||
stored[1]
|
.extra_data
|
||||||
.extra_data
|
.as_ref()
|
||||||
.as_ref()
|
.and_then(|value| value.get("dispatch_ref"))
|
||||||
.and_then(|value| value.get("dispatch_ref"))
|
.is_none());
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("normal-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2694,7 +2795,7 @@ mod tests {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.expect("ranking metadata should persist as object extra data");
|
.expect("ranking metadata should persist as object extra data");
|
||||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
assert!(extra_data.get("existing").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
extra_data.get("ranking_mode"),
|
extra_data.get("ranking_mode"),
|
||||||
Some(&json!("CacheAffinity"))
|
Some(&json!("CacheAffinity"))
|
||||||
@@ -2707,14 +2808,7 @@ mod tests {
|
|||||||
Some(&json!("cached_affinity"))
|
Some(&json!("cached_affinity"))
|
||||||
);
|
);
|
||||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||||
assert_eq!(
|
assert!(extra_data.get("dispatch_ref").is_none());
|
||||||
extra_data
|
|
||||||
.get("dispatch_ref")
|
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("ranked-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3052,7 +3146,7 @@ mod tests {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.expect("skipped ranking metadata should persist");
|
.expect("skipped ranking metadata should persist");
|
||||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
assert!(extra_data.get("existing").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
extra_data.get("ranking_mode"),
|
extra_data.get("ranking_mode"),
|
||||||
Some(&json!("CacheAffinity"))
|
Some(&json!("CacheAffinity"))
|
||||||
|
|||||||
@@ -278,13 +278,21 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||||
"chrome_136"
|
Value::Bool(true)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||||
|
Value::Bool(true)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||||
"chrome_136"
|
"chrome_136"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||||
|
"chrome_136"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||||
Value::Bool(true)
|
Value::Bool(true)
|
||||||
|
|||||||
@@ -251,6 +251,7 @@ mod tests {
|
|||||||
use aether_ai_serving::{
|
use aether_ai_serving::{
|
||||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||||
};
|
};
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
use aether_data::repository::{
|
use aether_data::repository::{
|
||||||
provider_catalog::InMemoryProviderCatalogReadRepository,
|
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||||
routing_profiles::InMemoryRoutingGroupRepository,
|
routing_profiles::InMemoryRoutingGroupRepository,
|
||||||
@@ -361,6 +362,8 @@ mod tests {
|
|||||||
candidate.key_internal_priority = 3;
|
candidate.key_internal_priority = 3;
|
||||||
candidate.key_global_priority_for_format = Some(2);
|
candidate.key_global_priority_for_format = Some(2);
|
||||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("group-1".to_string()),
|
group_id: Some("group-1".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "system_default".to_string(),
|
selection_source: "system_default".to_string(),
|
||||||
@@ -398,6 +401,8 @@ mod tests {
|
|||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
.with_data_state_for_tests(data_state);
|
.with_data_state_for_tests(data_state);
|
||||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("group-1".to_string()),
|
group_id: Some("group-1".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "system_default".to_string(),
|
selection_source: "system_default".to_string(),
|
||||||
@@ -433,6 +438,8 @@ mod tests {
|
|||||||
candidate.key_internal_priority = 3;
|
candidate.key_internal_priority = 3;
|
||||||
candidate.key_global_priority_for_format = Some(2);
|
candidate.key_global_priority_for_format = Some(2);
|
||||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("group-1".to_string()),
|
group_id: Some("group-1".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "system_default".to_string(),
|
selection_source: "system_default".to_string(),
|
||||||
@@ -566,6 +573,15 @@ mod tests {
|
|||||||
api_formats: Option<serde_json::Value>,
|
api_formats: Option<serde_json::Value>,
|
||||||
allowed_models: Option<serde_json::Value>,
|
allowed_models: Option<serde_json::Value>,
|
||||||
) -> StoredProviderCatalogKey {
|
) -> StoredProviderCatalogKey {
|
||||||
|
let credential_state = AppState::new()
|
||||||
|
.expect("credential state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
);
|
||||||
|
let encrypted_api_key = credential_state
|
||||||
|
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||||
|
.expect("api key should encrypt");
|
||||||
StoredProviderCatalogKey::new(
|
StoredProviderCatalogKey::new(
|
||||||
id.to_string(),
|
id.to_string(),
|
||||||
provider_id.to_string(),
|
provider_id.to_string(),
|
||||||
@@ -577,7 +593,7 @@ mod tests {
|
|||||||
.expect("key should build")
|
.expect("key should build")
|
||||||
.with_transport_fields(
|
.with_transport_fields(
|
||||||
api_formats,
|
api_formats,
|
||||||
"plain-upstream-key".to_string(),
|
encrypted_api_key,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
Some(json!({"openai:chat": 1})),
|
Some(json!({"openai:chat": 1})),
|
||||||
@@ -691,7 +707,7 @@ mod tests {
|
|||||||
let observed_at_unix_secs = current_unix_secs();
|
let observed_at_unix_secs = current_unix_secs();
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![
|
.with_system_config_values_for_tests(vec![
|
||||||
("provider_priority_mode".to_string(), json!("provider")),
|
("provider_priority_mode".to_string(), json!("provider")),
|
||||||
@@ -700,6 +716,7 @@ mod tests {
|
|||||||
serde_json::to_value(TunnelAttachmentRecord {
|
serde_json::to_value(TunnelAttachmentRecord {
|
||||||
gateway_instance_id: "gateway-b".to_string(),
|
gateway_instance_id: "gateway-b".to_string(),
|
||||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||||
|
tunnel_generation: "test-generation-remote".to_string(),
|
||||||
conn_count: 1,
|
conn_count: 1,
|
||||||
observed_at_unix_secs,
|
observed_at_unix_secs,
|
||||||
})
|
})
|
||||||
@@ -710,6 +727,7 @@ mod tests {
|
|||||||
serde_json::to_value(TunnelAttachmentRecord {
|
serde_json::to_value(TunnelAttachmentRecord {
|
||||||
gateway_instance_id: "gateway-a".to_string(),
|
gateway_instance_id: "gateway-a".to_string(),
|
||||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||||
|
tunnel_generation: "test-generation-local".to_string(),
|
||||||
conn_count: 1,
|
conn_count: 1,
|
||||||
observed_at_unix_secs,
|
observed_at_unix_secs,
|
||||||
})
|
})
|
||||||
@@ -768,7 +786,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -821,7 +839,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![(
|
.with_system_config_values_for_tests(vec![(
|
||||||
"scheduling_mode".to_string(),
|
"scheduling_mode".to_string(),
|
||||||
@@ -878,7 +896,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -953,7 +971,7 @@ mod tests {
|
|||||||
.expect("routing strategy should be created");
|
.expect("routing strategy should be created");
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_routing_group_repository_for_tests(routing_repository);
|
.with_routing_group_repository_for_tests(routing_repository);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
@@ -1015,7 +1033,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![(
|
.with_system_config_values_for_tests(vec![(
|
||||||
"provider_priority_mode".to_string(),
|
"provider_priority_mode".to_string(),
|
||||||
@@ -1081,7 +1099,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1134,7 +1152,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1208,7 +1226,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1288,7 +1306,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1364,7 +1382,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1431,7 +1449,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1514,7 +1532,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1579,7 +1597,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1668,7 +1686,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1754,7 +1772,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1851,7 +1869,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1956,7 +1974,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -2050,7 +2068,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
|
|||||||
@@ -93,6 +93,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
|||||||
&self,
|
&self,
|
||||||
candidate: Self::Candidate,
|
candidate: Self::Candidate,
|
||||||
) -> Self::Skipped {
|
) -> Self::Skipped {
|
||||||
|
warn!(
|
||||||
|
event_name = "local_candidate_skipped",
|
||||||
|
log_type = "event",
|
||||||
|
provider_id = %candidate.provider_id,
|
||||||
|
endpoint_id = %candidate.endpoint_id,
|
||||||
|
key_id = %candidate.key_id,
|
||||||
|
api_format = %candidate.endpoint_api_format,
|
||||||
|
skip_reason = "transport_snapshot_missing",
|
||||||
|
"local execution candidate skipped during planning"
|
||||||
|
);
|
||||||
SkippedLocalExecutionCandidate {
|
SkippedLocalExecutionCandidate {
|
||||||
candidate,
|
candidate,
|
||||||
skip_reason: "transport_snapshot_missing",
|
skip_reason: "transport_snapshot_missing",
|
||||||
@@ -145,6 +155,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
|||||||
transport: Self::Transport,
|
transport: Self::Transport,
|
||||||
skip_reason: &'static str,
|
skip_reason: &'static str,
|
||||||
) -> Self::Skipped {
|
) -> Self::Skipped {
|
||||||
|
warn!(
|
||||||
|
event_name = "local_candidate_skipped",
|
||||||
|
log_type = "event",
|
||||||
|
provider_id = %candidate.provider_id,
|
||||||
|
endpoint_id = %candidate.endpoint_id,
|
||||||
|
key_id = %candidate.key_id,
|
||||||
|
api_format = %candidate.endpoint_api_format,
|
||||||
|
skip_reason,
|
||||||
|
"local execution candidate skipped during planning"
|
||||||
|
);
|
||||||
SkippedLocalExecutionCandidate {
|
SkippedLocalExecutionCandidate {
|
||||||
candidate,
|
candidate,
|
||||||
skip_reason,
|
skip_reason,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use aether_routing_core::ResolvedRoutingPolicy;
|
|||||||
use aether_runtime::ConcurrencyPermit;
|
use aether_runtime::ConcurrencyPermit;
|
||||||
use aether_scheduler_core::{
|
use aether_scheduler_core::{
|
||||||
enumerate_minimal_candidate_selection_with_model_directives, normalize_api_format,
|
enumerate_minimal_candidate_selection_with_model_directives, normalize_api_format,
|
||||||
resolve_requested_global_model_name_with_model_directives_and_request_operation,
|
resolve_requested_global_model_name_with_reserved_global_model,
|
||||||
row_supports_requested_model_with_model_directives_and_request_operation,
|
row_supports_requested_model_with_model_directives_and_request_operation,
|
||||||
ClientSessionAffinity, EnumerateMinimalCandidateSelectionInput,
|
ClientSessionAffinity, EnumerateMinimalCandidateSelectionInput,
|
||||||
SchedulerMinimalCandidateSelectionCandidate,
|
SchedulerMinimalCandidateSelectionCandidate,
|
||||||
@@ -378,6 +378,7 @@ pub(crate) struct LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
requested_name_offsets: BTreeMap<String, u32>,
|
requested_name_offsets: BTreeMap<String, u32>,
|
||||||
scanned_rows_by_format: BTreeMap<String, u32>,
|
scanned_rows_by_format: BTreeMap<String, u32>,
|
||||||
resolved_global_model_names: BTreeMap<String, String>,
|
resolved_global_model_names: BTreeMap<String, String>,
|
||||||
|
reserved_global_model_names: BTreeMap<String, Option<String>>,
|
||||||
fallback_offsets: BTreeMap<String, u32>,
|
fallback_offsets: BTreeMap<String, u32>,
|
||||||
fallback_scan_epoch: u32,
|
fallback_scan_epoch: u32,
|
||||||
exhausted_api_formats: BTreeSet<String>,
|
exhausted_api_formats: BTreeSet<String>,
|
||||||
@@ -457,6 +458,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
requested_name_offsets: BTreeMap::new(),
|
requested_name_offsets: BTreeMap::new(),
|
||||||
scanned_rows_by_format: BTreeMap::new(),
|
scanned_rows_by_format: BTreeMap::new(),
|
||||||
resolved_global_model_names: BTreeMap::new(),
|
resolved_global_model_names: BTreeMap::new(),
|
||||||
|
reserved_global_model_names: BTreeMap::new(),
|
||||||
fallback_offsets: BTreeMap::new(),
|
fallback_offsets: BTreeMap::new(),
|
||||||
fallback_scan_epoch: 0,
|
fallback_scan_epoch: 0,
|
||||||
exhausted_api_formats: BTreeSet::new(),
|
exhausted_api_formats: BTreeSet::new(),
|
||||||
@@ -555,6 +557,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
self.requested_name_offsets.clear();
|
self.requested_name_offsets.clear();
|
||||||
self.scanned_rows_by_format.clear();
|
self.scanned_rows_by_format.clear();
|
||||||
self.resolved_global_model_names.clear();
|
self.resolved_global_model_names.clear();
|
||||||
|
self.reserved_global_model_names.clear();
|
||||||
self.fallback_offsets.clear();
|
self.fallback_offsets.clear();
|
||||||
self.fallback_scan_epoch = self.fallback_scan_epoch.wrapping_add(1);
|
self.fallback_scan_epoch = self.fallback_scan_epoch.wrapping_add(1);
|
||||||
self.exhausted_api_formats.clear();
|
self.exhausted_api_formats.clear();
|
||||||
@@ -1185,6 +1188,34 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
|| self.exhausted_api_formats.contains(&normalized_api_format)
|
|| self.exhausted_api_formats.contains(&normalized_api_format)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Global model names are a reserved routing namespace, so a request that
|
||||||
|
/// names one must not be answered by a provider whose own model merely
|
||||||
|
/// carries that name as an upstream alias. Cached per routing model: the
|
||||||
|
/// answer does not change between pages or API formats.
|
||||||
|
async fn reserved_global_model_name(
|
||||||
|
&mut self,
|
||||||
|
rows: &[StoredMinimalCandidateSelectionRow],
|
||||||
|
routing_model: &str,
|
||||||
|
) -> Result<Option<String>, GatewayError> {
|
||||||
|
if let Some(cached) = self.reserved_global_model_names.get(routing_model) {
|
||||||
|
return Ok(cached.clone());
|
||||||
|
}
|
||||||
|
let state = self.state;
|
||||||
|
let reserved_global_model_name =
|
||||||
|
crate::data::candidate_selection::resolve_reserved_global_model_name(
|
||||||
|
state.app().data.as_ref(),
|
||||||
|
rows,
|
||||||
|
routing_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
self.reserved_global_model_names.insert(
|
||||||
|
routing_model.to_string(),
|
||||||
|
reserved_global_model_name.clone(),
|
||||||
|
);
|
||||||
|
Ok(reserved_global_model_name)
|
||||||
|
}
|
||||||
|
|
||||||
async fn build_page_outcome_from_rows(
|
async fn build_page_outcome_from_rows(
|
||||||
&mut self,
|
&mut self,
|
||||||
candidate_api_format: &str,
|
candidate_api_format: &str,
|
||||||
@@ -1216,15 +1247,17 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
if let Some(value) = self.resolved_global_model_names.get(normalized_api_format) {
|
if let Some(value) = self.resolved_global_model_names.get(normalized_api_format) {
|
||||||
value.clone()
|
value.clone()
|
||||||
} else {
|
} else {
|
||||||
let Some(value) =
|
let reserved_global_model_name = self
|
||||||
resolve_requested_global_model_name_with_model_directives_and_request_operation(
|
.reserved_global_model_name(&rows, &routing_model)
|
||||||
&rows,
|
.await?;
|
||||||
&routing_model,
|
let Some(value) = resolve_requested_global_model_name_with_reserved_global_model(
|
||||||
normalized_api_format,
|
&rows,
|
||||||
false,
|
&routing_model,
|
||||||
self.request_operation.as_deref(),
|
normalized_api_format,
|
||||||
)
|
false,
|
||||||
else {
|
self.request_operation.as_deref(),
|
||||||
|
reserved_global_model_name.as_deref(),
|
||||||
|
) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
self.resolved_global_model_names
|
self.resolved_global_model_names
|
||||||
@@ -1473,7 +1506,9 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||||
|
use aether_data::repository::global_models::InMemoryGlobalModelReadRepository;
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
use aether_data::DataLayerError;
|
use aether_data::DataLayerError;
|
||||||
use aether_data_contracts::repository::candidate_selection::{
|
use aether_data_contracts::repository::candidate_selection::{
|
||||||
@@ -1481,6 +1516,9 @@ mod tests {
|
|||||||
StoredPoolKeyCandidateRowsByKeyIdsQuery, StoredPoolKeyCandidateRowsQuery,
|
StoredPoolKeyCandidateRowsByKeyIdsQuery, StoredPoolKeyCandidateRowsQuery,
|
||||||
StoredProviderModelMapping, StoredRequestedModelCandidateRowsQuery,
|
StoredProviderModelMapping, StoredRequestedModelCandidateRowsQuery,
|
||||||
};
|
};
|
||||||
|
use aether_data_contracts::repository::global_models::{
|
||||||
|
GlobalModelReadRepository, StoredPublicGlobalModel,
|
||||||
|
};
|
||||||
use aether_data_contracts::repository::provider_catalog::{
|
use aether_data_contracts::repository::provider_catalog::{
|
||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
@@ -1854,6 +1892,191 @@ mod tests {
|
|||||||
.is_none());
|
.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn routing_policy_excludes_group_disabled_providers_from_candidate_pages() {
|
||||||
|
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||||
|
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||||
|
standard_candidate_row("provider-disabled", "openai:chat", 0),
|
||||||
|
standard_candidate_row("provider-enabled", "openai:chat", 1),
|
||||||
|
]));
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("gateway state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||||
|
);
|
||||||
|
let auth_snapshot = unrestricted_auth_snapshot();
|
||||||
|
let model_directive_policy =
|
||||||
|
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
|
let config = serde_json::from_value(serde_json::json!({
|
||||||
|
"disabled_providers": ["provider-disabled"],
|
||||||
|
"model_policies": [{
|
||||||
|
"model": "*",
|
||||||
|
"allowed_providers": ["provider-disabled", "provider-enabled"]
|
||||||
|
}]
|
||||||
|
}))
|
||||||
|
.expect("routing config should parse");
|
||||||
|
let routing_policy = aether_routing_core::resolve_routing_policy(
|
||||||
|
&config,
|
||||||
|
aether_routing_core::RoutingPolicyInput {
|
||||||
|
group_id: Some("routing-group-1"),
|
||||||
|
group_version: Some(1),
|
||||||
|
selection_source: "test",
|
||||||
|
requested_model: "gpt-5",
|
||||||
|
resolved_model: "gpt-5",
|
||||||
|
api_format: "openai:chat",
|
||||||
|
user_id: None,
|
||||||
|
api_key_id: None,
|
||||||
|
headers: &serde_json::json!({}),
|
||||||
|
body: &serde_json::json!({}),
|
||||||
|
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("routing policy should resolve");
|
||||||
|
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
|
PlannerAppState::new(&app),
|
||||||
|
&model_directive_policy,
|
||||||
|
"openai:chat",
|
||||||
|
"gpt-5",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
&auth_snapshot,
|
||||||
|
Some(&routing_policy),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let page = cursor
|
||||||
|
.next_page()
|
||||||
|
.await
|
||||||
|
.expect("routing candidate scan should succeed")
|
||||||
|
.expect("the enabled provider should remain");
|
||||||
|
assert_eq!(
|
||||||
|
page.candidates
|
||||||
|
.iter()
|
||||||
|
.map(|candidate| candidate.provider_id.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["provider-enabled"]
|
||||||
|
);
|
||||||
|
assert!(cursor
|
||||||
|
.next_page()
|
||||||
|
.await
|
||||||
|
.expect("routing scan should finish")
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn model_provider_enablement_filters_candidate_pages_without_affecting_other_models() {
|
||||||
|
let mut rows = Vec::new();
|
||||||
|
for model in ["model-a", "model-b", "model-c"] {
|
||||||
|
for (provider, priority) in [
|
||||||
|
("provider-legacy-disabled", 0),
|
||||||
|
("provider-model-disabled", 1),
|
||||||
|
("provider-other", 2),
|
||||||
|
("provider-inactive", 3),
|
||||||
|
] {
|
||||||
|
let mut row = standard_candidate_row(provider, "openai:chat", priority);
|
||||||
|
row.global_model_id = format!("global-{model}");
|
||||||
|
row.global_model_name = model.into();
|
||||||
|
row.model_provider_model_name = model.into();
|
||||||
|
row.model_id = format!("{provider}-{model}");
|
||||||
|
row.provider_is_active = provider != "provider-inactive";
|
||||||
|
rows.push(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||||
|
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows));
|
||||||
|
let app = AppState::new().unwrap().with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||||
|
);
|
||||||
|
let auth = unrestricted_auth_snapshot();
|
||||||
|
let directives = crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
|
let config = serde_json::from_value(serde_json::json!({
|
||||||
|
"disabled_providers": ["provider-legacy-disabled"],
|
||||||
|
"model_policies": [
|
||||||
|
{ "model": "model-a", "provider_enabled_overrides": {
|
||||||
|
"provider-model-disabled": false, "provider-inactive": true
|
||||||
|
} },
|
||||||
|
{ "model": "model-b", "provider_enabled_overrides": {
|
||||||
|
"provider-legacy-disabled": true, "provider-inactive": true
|
||||||
|
} }
|
||||||
|
],
|
||||||
|
"rules": [{ "id": "legacy-allowlist", "actions": [{
|
||||||
|
"type": "restrict_providers", "provider_ids": [
|
||||||
|
"provider-legacy-disabled", "provider-model-disabled", "provider-other", "provider-inactive"
|
||||||
|
]
|
||||||
|
}] }]
|
||||||
|
})).unwrap();
|
||||||
|
// Revisit A after B to exercise candidate caches shared by the app.
|
||||||
|
for (model, expected) in [
|
||||||
|
("model-a", vec!["provider-other"]),
|
||||||
|
(
|
||||||
|
"model-b",
|
||||||
|
vec![
|
||||||
|
"provider-legacy-disabled",
|
||||||
|
"provider-model-disabled",
|
||||||
|
"provider-other",
|
||||||
|
],
|
||||||
|
),
|
||||||
|
("model-c", vec!["provider-model-disabled", "provider-other"]),
|
||||||
|
("model-a", vec!["provider-other"]),
|
||||||
|
] {
|
||||||
|
let policy = aether_routing_core::resolve_routing_policy(
|
||||||
|
&config,
|
||||||
|
aether_routing_core::RoutingPolicyInput {
|
||||||
|
group_id: Some("group-1"),
|
||||||
|
group_version: Some(1),
|
||||||
|
selection_source: "test",
|
||||||
|
requested_model: model,
|
||||||
|
resolved_model: model,
|
||||||
|
api_format: "openai:chat",
|
||||||
|
user_id: None,
|
||||||
|
api_key_id: None,
|
||||||
|
headers: &serde_json::json!({}),
|
||||||
|
body: &serde_json::json!({}),
|
||||||
|
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
|
PlannerAppState::new(&app),
|
||||||
|
&directives,
|
||||||
|
"openai:chat",
|
||||||
|
model,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
&auth,
|
||||||
|
Some(&policy),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut providers = Vec::new();
|
||||||
|
while let Some(page) = cursor.next_page().await.unwrap() {
|
||||||
|
providers.extend(
|
||||||
|
page.candidates
|
||||||
|
.into_iter()
|
||||||
|
.map(|candidate| candidate.provider_id),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
providers.sort();
|
||||||
|
assert_eq!(
|
||||||
|
providers, expected,
|
||||||
|
"provider enablement must remain isolated for {model}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn routing_policy_collects_candidate_pages_before_final_ranking() {
|
async fn routing_policy_collects_candidate_pages_before_final_ranking() {
|
||||||
let rows = (0..300)
|
let rows = (0..300)
|
||||||
@@ -1876,6 +2099,8 @@ mod tests {
|
|||||||
let model_directive_policy =
|
let model_directive_policy =
|
||||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
let routing_policy = ResolvedRoutingPolicy {
|
let routing_policy = ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("routing-group-1".to_string()),
|
group_id: Some("routing-group-1".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "test".to_string(),
|
selection_source: "test".to_string(),
|
||||||
@@ -1941,6 +2166,8 @@ mod tests {
|
|||||||
let model_directive_policy =
|
let model_directive_policy =
|
||||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
let routing_policy = ResolvedRoutingPolicy {
|
let routing_policy = ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("routing-group-fallback".to_string()),
|
group_id: Some("routing-group-fallback".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "test".to_string(),
|
selection_source: "test".to_string(),
|
||||||
@@ -2089,6 +2316,96 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn public_global_model(name: &str) -> StoredPublicGlobalModel {
|
||||||
|
StoredPublicGlobalModel {
|
||||||
|
id: format!("global-model-{name}"),
|
||||||
|
name: name.to_string(),
|
||||||
|
display_name: None,
|
||||||
|
is_active: true,
|
||||||
|
default_price_per_request: None,
|
||||||
|
default_tiered_pricing: None,
|
||||||
|
supported_capabilities: None,
|
||||||
|
config: None,
|
||||||
|
usage_count: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cursor provider reaches its upstream under a name that belongs to another
|
||||||
|
/// global model. A `claude:messages` client asking for `gemini-3.8-flash` has to
|
||||||
|
/// land on the provider bound to that global model — format conversion and all —
|
||||||
|
/// rather than on the one that only borrows the name on the way out, which is the
|
||||||
|
/// one an API-format-ordered scan reaches first.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn paged_preselection_keeps_a_global_model_name_from_a_provider_alias() {
|
||||||
|
let mut aliasing = standard_candidate_row("ursor", "claude:messages", 1);
|
||||||
|
aliasing.global_model_id = "global-model-gemini-3.8-flash-cursor".to_string();
|
||||||
|
aliasing.global_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||||
|
aliasing.model_provider_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||||
|
aliasing.model_provider_model_mappings = Some(vec![StoredProviderModelMapping {
|
||||||
|
name: "gemini-3.8-flash".to_string(),
|
||||||
|
priority: 1,
|
||||||
|
api_formats: None,
|
||||||
|
endpoint_ids: None,
|
||||||
|
operations: None,
|
||||||
|
}]);
|
||||||
|
|
||||||
|
let mut bound = standard_candidate_row("anti", "gemini:generate_content", 2);
|
||||||
|
bound.global_model_id = "global-model-gemini-3.8-flash".to_string();
|
||||||
|
bound.global_model_name = "gemini-3.8-flash".to_string();
|
||||||
|
bound.model_provider_model_name = "gemini-3.8-flash".to_string();
|
||||||
|
|
||||||
|
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||||
|
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||||
|
aliasing, bound,
|
||||||
|
]));
|
||||||
|
let global_models: Arc<dyn GlobalModelReadRepository> =
|
||||||
|
Arc::new(InMemoryGlobalModelReadRepository::seed([
|
||||||
|
public_global_model("gemini-3.8-flash"),
|
||||||
|
public_global_model("gemini-3.8-flash-cursor"),
|
||||||
|
]));
|
||||||
|
let data_state =
|
||||||
|
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository)
|
||||||
|
.with_global_model_reader(global_models);
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("gateway state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let auth_snapshot = unrestricted_auth_snapshot();
|
||||||
|
let model_directive_policy =
|
||||||
|
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
|
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
|
PlannerAppState::new(&app),
|
||||||
|
&model_directive_policy,
|
||||||
|
"claude:messages",
|
||||||
|
"gemini-3.8-flash",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
&auth_snapshot,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let page = cursor
|
||||||
|
.next_page()
|
||||||
|
.await
|
||||||
|
.expect("preselection should succeed")
|
||||||
|
.expect("the bound provider should still be reachable");
|
||||||
|
|
||||||
|
assert_eq!(page.candidates.len(), 1);
|
||||||
|
assert_eq!(page.candidates[0].provider_name, "anti");
|
||||||
|
assert_eq!(page.candidates[0].global_model_name, "gemini-3.8-flash");
|
||||||
|
assert_eq!(
|
||||||
|
page.candidates[0].endpoint_api_format,
|
||||||
|
"gemini:generate_content"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn standard_candidate_row(
|
fn standard_candidate_row(
|
||||||
provider_id: &str,
|
provider_id: &str,
|
||||||
api_format: &str,
|
api_format: &str,
|
||||||
@@ -2174,6 +2491,19 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.expect("endpoint transport should build");
|
.expect("endpoint transport should build");
|
||||||
|
let credential_state = AppState::new()
|
||||||
|
.expect("credential state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
);
|
||||||
|
let encrypted_api_key = credential_state
|
||||||
|
.seal_provider_catalog_key_api_key(
|
||||||
|
row.provider_id.as_str(),
|
||||||
|
row.key_id.as_str(),
|
||||||
|
"plain-upstream-key",
|
||||||
|
)
|
||||||
|
.expect("api key should encrypt");
|
||||||
let key = StoredProviderCatalogKey::new(
|
let key = StoredProviderCatalogKey::new(
|
||||||
row.key_id.clone(),
|
row.key_id.clone(),
|
||||||
row.provider_id.clone(),
|
row.provider_id.clone(),
|
||||||
@@ -2185,7 +2515,7 @@ mod tests {
|
|||||||
.expect("key should build")
|
.expect("key should build")
|
||||||
.with_transport_fields(
|
.with_transport_fields(
|
||||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||||
"plain-upstream-key".to_string(),
|
encrypted_api_key,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -2540,7 +2870,7 @@ mod tests {
|
|||||||
provider_repository,
|
provider_repository,
|
||||||
candidate_repository,
|
candidate_repository,
|
||||||
)
|
)
|
||||||
.with_encryption_key_for_tests("development-key");
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||||
let app = AppState::new()
|
let app = AppState::new()
|
||||||
.expect("gateway state should build")
|
.expect("gateway state should build")
|
||||||
.with_data_state_for_tests(data_state);
|
.with_data_state_for_tests(data_state);
|
||||||
@@ -2660,7 +2990,7 @@ mod tests {
|
|||||||
provider_repository,
|
provider_repository,
|
||||||
candidate_repository,
|
candidate_repository,
|
||||||
)
|
)
|
||||||
.with_encryption_key_for_tests("development-key")
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
// Legacy keys deliberately disagree with the routing policy: the
|
// Legacy keys deliberately disagree with the routing policy: the
|
||||||
// resolved policy must be the only source of scheduler ordering.
|
// resolved policy must be the only source of scheduler ordering.
|
||||||
.with_system_config_values_for_tests([
|
.with_system_config_values_for_tests([
|
||||||
@@ -2680,6 +3010,8 @@ mod tests {
|
|||||||
let model_directive_policy =
|
let model_directive_policy =
|
||||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||||
let routing_policy = ResolvedRoutingPolicy {
|
let routing_policy = ResolvedRoutingPolicy {
|
||||||
|
billing_multiplier: 1.0,
|
||||||
|
group_name: None,
|
||||||
group_id: Some("routing-group-codex-first".to_string()),
|
group_id: Some("routing-group-codex-first".to_string()),
|
||||||
group_version: Some(1),
|
group_version: Some(1),
|
||||||
selection_source: "test".to_string(),
|
selection_source: "test".to_string(),
|
||||||
|
|||||||
@@ -19,12 +19,12 @@ pub(crate) use crate::ai_serving::{
|
|||||||
GEMINI_FILES_UPLOAD_PLAN_KIND, GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
GEMINI_FILES_UPLOAD_PLAN_KIND, GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||||
GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND,
|
GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND,
|
||||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND,
|
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND,
|
||||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||||
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
||||||
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||||
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) use aether_ai_serving::AiRequestedModelFamily as RequestedModelFamily;
|
pub(crate) use aether_ai_serving::AiRequestedModelFamily as RequestedModelFamily;
|
||||||
|
|||||||
@@ -6,12 +6,13 @@ use crate::ai_serving::planner::common::{
|
|||||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND, GEMINI_FILES_LIST_PLAN_KIND,
|
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND, GEMINI_FILES_LIST_PLAN_KIND,
|
||||||
GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND, GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND, GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||||
OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_EMBEDDING_SYNC_PLAN_KIND,
|
OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_EMBEDDING_SYNC_PLAN_KIND,
|
||||||
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||||
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||||
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
||||||
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||||
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
||||||
|
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::plan_builders::{
|
use crate::ai_serving::planner::plan_builders::{
|
||||||
build_gemini_stream_plan_from_decision, build_gemini_sync_plan_from_decision,
|
build_gemini_stream_plan_from_decision, build_gemini_sync_plan_from_decision,
|
||||||
@@ -101,7 +102,9 @@ fn build_sync_plan_payload_from_decision(
|
|||||||
OPENAI_RESPONSES_SYNC_PLAN_KIND => {
|
OPENAI_RESPONSES_SYNC_PLAN_KIND => {
|
||||||
build_openai_responses_sync_plan_from_decision(parts, body_json, payload, false)?
|
build_openai_responses_sync_plan_from_decision(parts, body_json, payload, false)?
|
||||||
}
|
}
|
||||||
OPENAI_IMAGE_SYNC_PLAN_KIND | OPENAI_SEARCH_SYNC_PLAN_KIND => {
|
OPENAI_IMAGE_SYNC_PLAN_KIND
|
||||||
|
| OPENAI_SEARCH_SYNC_PLAN_KIND
|
||||||
|
| OPENAI_MEMORIES_SYNC_PLAN_KIND => {
|
||||||
build_passthrough_sync_plan_from_decision(parts, payload)?
|
build_passthrough_sync_plan_from_decision(parts, payload)?
|
||||||
}
|
}
|
||||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND => {
|
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND => {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
|||||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = state
|
let Some(user_id) = decision
|
||||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
.auth_context
|
||||||
.await?;
|
.as_ref()
|
||||||
|
.filter(|auth_context| auth_context.access_allowed)
|
||||||
|
.map(|auth_context| auth_context.user_id.trim())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
else {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
};
|
||||||
|
if state
|
||||||
|
.hydrate_video_task_for_route_for_user(
|
||||||
|
decision.route_family.as_deref(),
|
||||||
|
parts.uri.path(),
|
||||||
|
user_id,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
|
||||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
let Some(action) = state
|
||||||
parts.uri.path(),
|
.video_tasks
|
||||||
parts.uri.query(),
|
.prepare_openai_content_stream_action_for_user(
|
||||||
trace_id,
|
parts.uri.path(),
|
||||||
) else {
|
parts.uri.query(),
|
||||||
return Ok(None);
|
trace_id,
|
||||||
|
user_id,
|
||||||
|
)
|
||||||
|
else {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
|
|
||||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
|||||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||||
GatewayControlDecision,
|
GatewayControlDecision,
|
||||||
};
|
};
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = state
|
|
||||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let auth_context = resolve_execution_runtime_auth_context(
|
let auth_context = resolve_execution_runtime_auth_context(
|
||||||
state,
|
state,
|
||||||
decision,
|
decision,
|
||||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let Some(auth_context) = auth_context else {
|
let Some(auth_context) = auth_context else {
|
||||||
return Ok(None);
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
if state
|
||||||
|
.hydrate_video_task_for_route_for_user(
|
||||||
|
decision.route_family.as_deref(),
|
||||||
|
parts.uri.path(),
|
||||||
|
&auth_context.user_id,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||||
plan_kind,
|
plan_kind,
|
||||||
parts.uri.path(),
|
parts.uri.path(),
|
||||||
Some(body_json),
|
Some(body_json),
|
||||||
Some(&auth_context),
|
Some(&auth_context),
|
||||||
trace_id,
|
trace_id,
|
||||||
) else {
|
) else {
|
||||||
return Ok(None);
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
|
|
||||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
upstream_base_url = ?upstream_base_url,
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
upstream_url = %plan.url,
|
|
||||||
"gateway built local video follow-up sync decision payload"
|
"gateway built local video follow-up sync decision payload"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -13,11 +13,11 @@ use http::{HeaderMap, HeaderName, HeaderValue};
|
|||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
||||||
use crate::ai_serving::transport::CodexFingerprintConvergenceContext;
|
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||||
CODEX_RESPONSES_LITE_HEADER,
|
CODEX_RESPONSES_LITE_HEADER, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||||
};
|
};
|
||||||
use crate::cache::CacheLoadObserver;
|
use crate::cache::CacheLoadObserver;
|
||||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
|||||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||||
|
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||||
|
"routing provider request violates provider contract";
|
||||||
|
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||||
|
"invalid provider request headers in routing mutation";
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||||
@@ -56,7 +60,7 @@ pub(crate) struct LocalRequestedModelDecisionInput {
|
|||||||
pub(crate) client_surface: Option<ClientSurface>,
|
pub(crate) client_surface: Option<ClientSurface>,
|
||||||
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
||||||
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
||||||
pub(crate) codex_fingerprint_context: Option<CodexFingerprintConvergenceContext>,
|
pub(crate) provider_outbound_context: Option<ProviderOutboundRequestContext>,
|
||||||
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
||||||
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
||||||
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
||||||
@@ -119,6 +123,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||||
websocket_continuation: bool,
|
websocket_continuation: bool,
|
||||||
) -> Result<(), GatewayError> {
|
) -> Result<(), GatewayError> {
|
||||||
|
let native_memories = decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND);
|
||||||
let provider_api_format = decision
|
let provider_api_format = decision
|
||||||
.provider_api_format
|
.provider_api_format
|
||||||
.clone()
|
.clone()
|
||||||
@@ -146,7 +151,12 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
input.requested_model.as_str(),
|
input.requested_model.as_str(),
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
if native_memories {
|
||||||
|
decision
|
||||||
|
.provider_request_headers
|
||||||
|
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||||
|
} else {
|
||||||
|
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||||
&mut decision.provider_request_headers,
|
&mut decision.provider_request_headers,
|
||||||
decision.provider_request_body.as_ref(),
|
decision.provider_request_body.as_ref(),
|
||||||
provider_type.as_str(),
|
provider_type.as_str(),
|
||||||
@@ -155,6 +165,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
input.requested_model.as_str(),
|
input.requested_model.as_str(),
|
||||||
model_capabilities.as_ref(),
|
model_capabilities.as_ref(),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let Some(context) = input.routing_context.as_ref() else {
|
let Some(context) = input.routing_context.as_ref() else {
|
||||||
// Cache identity headers are projected only at the terminal boundary. Any non-empty
|
// Cache identity headers are projected only at the terminal boundary. Any non-empty
|
||||||
@@ -168,7 +179,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_api_format.as_str(),
|
provider_api_format.as_str(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -231,7 +242,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_api_format.as_str(),
|
provider_api_format.as_str(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -256,7 +267,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_headers.insert(HeaderName::from_static(name), value);
|
provider_headers.insert(HeaderName::from_static(name), value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if original_provider_request_body.is_some() {
|
if original_provider_request_body.is_some() && !native_memories {
|
||||||
let provider_model = provider_request_body
|
let provider_model = provider_request_body
|
||||||
.get("model")
|
.get("model")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -312,10 +323,13 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.map_err(|violation| GatewayError::Client {
|
.map_err(|_| invalid_routing_provider_contract())?;
|
||||||
status: StatusCode::BAD_REQUEST,
|
}
|
||||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
if native_memories {
|
||||||
})?;
|
crate::ai_serving::transport::enforce_same_format_provider_api_operation_body_policy(
|
||||||
|
&mut provider_request_body,
|
||||||
|
Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let provider_model = provider_request_body
|
let provider_model = provider_request_body
|
||||||
.get("model")
|
.get("model")
|
||||||
@@ -338,7 +352,11 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_type.as_str(),
|
provider_type.as_str(),
|
||||||
provider_api_format.as_str(),
|
provider_api_format.as_str(),
|
||||||
);
|
);
|
||||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
if native_memories {
|
||||||
|
provider_request_headers
|
||||||
|
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||||
|
} else {
|
||||||
|
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||||
&mut provider_request_headers,
|
&mut provider_request_headers,
|
||||||
Some(&provider_request_body),
|
Some(&provider_request_body),
|
||||||
provider_type.as_str(),
|
provider_type.as_str(),
|
||||||
@@ -347,6 +365,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
input.requested_model.as_str(),
|
input.requested_model.as_str(),
|
||||||
model_capabilities.as_ref(),
|
model_capabilities.as_ref(),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
crate::ai_serving::apply_codex_openai_compact_terminal_headers(
|
crate::ai_serving::apply_codex_openai_compact_terminal_headers(
|
||||||
&mut provider_request_headers,
|
&mut provider_request_headers,
|
||||||
provider_type.as_str(),
|
provider_type.as_str(),
|
||||||
@@ -357,7 +376,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
if original_provider_request_body.is_some() {
|
if original_provider_request_body.is_some() {
|
||||||
decision.provider_request_body = Some(provider_request_body);
|
decision.provider_request_body = Some(provider_request_body);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -367,7 +386,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn apply_codex_fingerprint_convergence_to_decision(
|
fn apply_provider_outbound_request_policies_to_decision(
|
||||||
input: &LocalRequestedModelDecisionInput,
|
input: &LocalRequestedModelDecisionInput,
|
||||||
decision: &mut AiExecutionDecision,
|
decision: &mut AiExecutionDecision,
|
||||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||||
@@ -378,17 +397,26 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
|||||||
else {
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Some(context) = input.codex_fingerprint_context.as_ref() else {
|
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let applied = crate::ai_serving::transport::apply_codex_fingerprint_convergence_with_context(
|
let native_context;
|
||||||
|
let context = if decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND) {
|
||||||
|
native_context = context
|
||||||
|
.clone()
|
||||||
|
.with_api_operation(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize);
|
||||||
|
&native_context
|
||||||
|
} else {
|
||||||
|
context
|
||||||
|
};
|
||||||
|
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||||
transport,
|
transport,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
context,
|
context,
|
||||||
&mut decision.provider_request_headers,
|
&mut decision.provider_request_headers,
|
||||||
provider_request_body,
|
provider_request_body,
|
||||||
);
|
);
|
||||||
if applied {
|
if results.iter().any(|result| result.was_applied()) {
|
||||||
decision.prompt_cache_key = provider_request_body
|
decision.prompt_cache_key = provider_request_body
|
||||||
.get("prompt_cache_key")
|
.get("prompt_cache_key")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -396,6 +424,31 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
|||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
.map(ToOwned::to_owned);
|
.map(ToOwned::to_owned);
|
||||||
}
|
}
|
||||||
|
if results.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for result in &results {
|
||||||
|
tracing::debug!(
|
||||||
|
event_name = "provider_outbound_policy_evaluated",
|
||||||
|
log_type = "event",
|
||||||
|
policy = ?result.policy,
|
||||||
|
outcome = ?result.outcome,
|
||||||
|
reason = ?result.reason,
|
||||||
|
mutation_scope = ?result.mutation_scope,
|
||||||
|
identity_scope = ?result.identity_scope,
|
||||||
|
"provider outbound request policy evaluated"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let Some(serde_json::Value::Object(report_context)) = decision.report_context.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
report_context.insert(
|
||||||
|
"provider_outbound_policies".to_string(),
|
||||||
|
serde_json::json!({
|
||||||
|
"schema_version": 1,
|
||||||
|
"results": results,
|
||||||
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
||||||
@@ -484,7 +537,7 @@ pub(crate) fn build_local_requested_model_decision_input(
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -499,27 +552,45 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
client_api_format: &str,
|
client_api_format: &str,
|
||||||
) -> Result<(), GatewayError> {
|
) -> Result<(), GatewayError> {
|
||||||
input.codex_fingerprint_context =
|
input.provider_outbound_context =
|
||||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||||
|
let preferred_group = if explicit_group.is_none() && !input.auth_context.api_key_is_standalone {
|
||||||
|
state
|
||||||
|
.read_auth_api_key_feature_settings(
|
||||||
|
&input.auth_context.user_id,
|
||||||
|
&input.auth_context.api_key_id,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|settings| settings.get("routing_group_id"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(str::to_owned)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let selected_group = match state.routing_group_read_repository() {
|
let selected_group = match state.routing_group_read_repository() {
|
||||||
Some(repository) => {
|
Some(repository) => {
|
||||||
// Explicit non-default groups are authorized against principal
|
// Explicit non-default groups are authorized against principal
|
||||||
// bindings, so both selection and its cache key must retain the
|
// bindings, so both selection and its cache key must retain the
|
||||||
// caller context. Only the implicit no-binding system-default
|
// caller context. Only the implicit no-binding system-default
|
||||||
// path is global and can skip the membership lookup.
|
// path is global and can skip the membership lookup.
|
||||||
let principal_context_required = if explicit_group.is_some() {
|
let principal_context_required =
|
||||||
true
|
if explicit_group.is_some() || preferred_group.is_some() {
|
||||||
} else {
|
true
|
||||||
repository
|
} else {
|
||||||
.has_any_routing_group_binding()
|
repository
|
||||||
.await
|
.has_any_routing_group_binding()
|
||||||
.map_err(|error| {
|
.await
|
||||||
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
.map_err(|error| {
|
||||||
error.to_string(),
|
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
||||||
))
|
error.to_string(),
|
||||||
})?
|
))
|
||||||
};
|
})?
|
||||||
|
};
|
||||||
let user_group_ids = if principal_context_required {
|
let user_group_ids = if principal_context_required {
|
||||||
let user_groups_lookup_started_at = std::time::Instant::now();
|
let user_groups_lookup_started_at = std::time::Instant::now();
|
||||||
let user_groups = state
|
let user_groups = state
|
||||||
@@ -542,6 +613,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
principal_context_required.then(|| input.auth_context.api_key_id.clone());
|
principal_context_required.then(|| input.auth_context.api_key_id.clone());
|
||||||
let selection_cache_key = routing_group_selection_cache_key(
|
let selection_cache_key = routing_group_selection_cache_key(
|
||||||
explicit_group.as_deref(),
|
explicit_group.as_deref(),
|
||||||
|
preferred_group.as_deref(),
|
||||||
selection_user_id.as_deref(),
|
selection_user_id.as_deref(),
|
||||||
selection_api_key_id.as_deref(),
|
selection_api_key_id.as_deref(),
|
||||||
&user_group_ids,
|
&user_group_ids,
|
||||||
@@ -559,6 +631,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
repository.as_ref(),
|
repository.as_ref(),
|
||||||
GatewayRoutingSelectionInput {
|
GatewayRoutingSelectionInput {
|
||||||
explicit_group: explicit_group.as_deref(),
|
explicit_group: explicit_group.as_deref(),
|
||||||
|
preferred_group: preferred_group.as_deref(),
|
||||||
user_id: selection_user_id.as_deref(),
|
user_id: selection_user_id.as_deref(),
|
||||||
api_key_id: selection_api_key_id.as_deref(),
|
api_key_id: selection_api_key_id.as_deref(),
|
||||||
user_group_ids: &user_group_ids,
|
user_group_ids: &user_group_ids,
|
||||||
@@ -575,6 +648,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
|| {
|
|| {
|
||||||
let repository = repository.clone();
|
let repository = repository.clone();
|
||||||
let explicit_group = explicit_group.clone();
|
let explicit_group = explicit_group.clone();
|
||||||
|
let preferred_group = preferred_group.clone();
|
||||||
let user_id = selection_user_id.clone();
|
let user_id = selection_user_id.clone();
|
||||||
let api_key_id = selection_api_key_id.clone();
|
let api_key_id = selection_api_key_id.clone();
|
||||||
let user_group_ids = user_group_ids.clone();
|
let user_group_ids = user_group_ids.clone();
|
||||||
@@ -584,6 +658,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
repository.as_ref(),
|
repository.as_ref(),
|
||||||
GatewayRoutingSelectionInput {
|
GatewayRoutingSelectionInput {
|
||||||
explicit_group: explicit_group.as_deref(),
|
explicit_group: explicit_group.as_deref(),
|
||||||
|
preferred_group: preferred_group.as_deref(),
|
||||||
user_id: user_id.as_deref(),
|
user_id: user_id.as_deref(),
|
||||||
api_key_id: api_key_id.as_deref(),
|
api_key_id: api_key_id.as_deref(),
|
||||||
user_group_ids: &user_group_ids,
|
user_group_ids: &user_group_ids,
|
||||||
@@ -609,6 +684,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
selection.group.map(|group| {
|
selection.group.map(|group| {
|
||||||
(
|
(
|
||||||
Some(group.id),
|
Some(group.id),
|
||||||
|
group.name,
|
||||||
Some(group.version),
|
Some(group.version),
|
||||||
group.config_json,
|
group.config_json,
|
||||||
selection.source,
|
selection.source,
|
||||||
@@ -616,13 +692,14 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
if explicit_group
|
if let Some(requested_group) = explicit_group
|
||||||
|
.or(preferred_group)
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
.is_some_and(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
{
|
{
|
||||||
return Err(routing_selection_error(
|
return Err(routing_selection_error(
|
||||||
GatewayRoutingSelectionError::NotFound(explicit_group.unwrap_or_default()),
|
GatewayRoutingSelectionError::NotFound(requested_group.to_string()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
return Err(routing_selection_error(
|
return Err(routing_selection_error(
|
||||||
@@ -631,7 +708,8 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some((group_id, group_version, group_config_json, selection_source)) = selected_group
|
let Some((group_id, group_name, group_version, group_config_json, selection_source)) =
|
||||||
|
selected_group
|
||||||
else {
|
else {
|
||||||
return Err(routing_selection_error(
|
return Err(routing_selection_error(
|
||||||
GatewayRoutingSelectionError::NoDefault,
|
GatewayRoutingSelectionError::NoDefault,
|
||||||
@@ -648,6 +726,12 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
&group_config_json,
|
&group_config_json,
|
||||||
selection_source.as_str(),
|
selection_source.as_str(),
|
||||||
)? {
|
)? {
|
||||||
|
if let Some(policy) = input.routing_policy.as_mut() {
|
||||||
|
policy.group_name = Some(group_name.clone());
|
||||||
|
}
|
||||||
|
if let Some(trace) = input.routing_trace_seed.as_mut() {
|
||||||
|
trace.group_name = Some(group_name);
|
||||||
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -733,6 +817,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
final_policy_resolve_started_at.elapsed().as_millis() as u64,
|
final_policy_resolve_started_at.elapsed().as_millis() as u64,
|
||||||
);
|
);
|
||||||
final_policy.mutation_plan = policy.mutation_plan.clone();
|
final_policy.mutation_plan = policy.mutation_plan.clone();
|
||||||
|
final_policy.group_name = Some(group_name);
|
||||||
input.routing_trace_seed = Some(build_routing_trace_seed(&final_policy, client_api_format));
|
input.routing_trace_seed = Some(build_routing_trace_seed(&final_policy, client_api_format));
|
||||||
input.routing_policy = Some(final_policy);
|
input.routing_policy = Some(final_policy);
|
||||||
input.routing_context = Some(LocalRoutingRequestContext {
|
input.routing_context = Some(LocalRoutingRequestContext {
|
||||||
@@ -863,10 +948,32 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
|||||||
status: StatusCode::SERVICE_UNAVAILABLE,
|
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||||
message: "no enabled routing strategy is configured for this request".to_string(),
|
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||||
},
|
},
|
||||||
error => GatewayError::Client {
|
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||||
status: StatusCode::FORBIDDEN,
|
status: StatusCode::FORBIDDEN,
|
||||||
message: error.to_string(),
|
message: "requested routing group was not found".to_string(),
|
||||||
},
|
},
|
||||||
|
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
message: "requested routing group is not enabled".to_string(),
|
||||||
|
},
|
||||||
|
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
message: "requested routing group is not allowed for this principal".to_string(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_routing_provider_contract() -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_routing_provider_headers() -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -891,6 +998,7 @@ fn routing_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<Stri
|
|||||||
|
|
||||||
fn routing_group_selection_cache_key(
|
fn routing_group_selection_cache_key(
|
||||||
explicit_group: Option<&str>,
|
explicit_group: Option<&str>,
|
||||||
|
preferred_group: Option<&str>,
|
||||||
user_id: Option<&str>,
|
user_id: Option<&str>,
|
||||||
api_key_id: Option<&str>,
|
api_key_id: Option<&str>,
|
||||||
user_group_ids: &[String],
|
user_group_ids: &[String],
|
||||||
@@ -901,8 +1009,9 @@ fn routing_group_selection_cache_key(
|
|||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(",");
|
.join(",");
|
||||||
format!(
|
format!(
|
||||||
"v1|explicit={}|user={}|api_key={}|groups={}",
|
"v2|explicit={}|preferred={}|user={}|api_key={}|groups={}",
|
||||||
escape_cache_key_part(explicit_group.unwrap_or_default()),
|
escape_cache_key_part(explicit_group.unwrap_or_default()),
|
||||||
|
escape_cache_key_part(preferred_group.unwrap_or_default()),
|
||||||
escape_cache_key_part(user_id.unwrap_or_default()),
|
escape_cache_key_part(user_id.unwrap_or_default()),
|
||||||
escape_cache_key_part(api_key_id.unwrap_or_default()),
|
escape_cache_key_part(api_key_id.unwrap_or_default()),
|
||||||
groups
|
groups
|
||||||
@@ -921,14 +1030,9 @@ fn btree_headers_to_header_map(
|
|||||||
) -> Result<HeaderMap, GatewayError> {
|
) -> Result<HeaderMap, GatewayError> {
|
||||||
let mut output = HeaderMap::new();
|
let mut output = HeaderMap::new();
|
||||||
for (name, value) in headers {
|
for (name, value) in headers {
|
||||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
let name = HeaderName::from_bytes(name.as_bytes())
|
||||||
status: StatusCode::BAD_REQUEST,
|
.map_err(|_| invalid_routing_provider_headers())?;
|
||||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||||
})?;
|
|
||||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
|
||||||
status: StatusCode::BAD_REQUEST,
|
|
||||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
|
||||||
})?;
|
|
||||||
output.insert(name, value);
|
output.insert(name, value);
|
||||||
}
|
}
|
||||||
Ok(output)
|
Ok(output)
|
||||||
@@ -1105,10 +1209,13 @@ mod tests {
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use aether_data::repository::auth::{
|
||||||
|
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||||
|
};
|
||||||
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
|
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
|
||||||
use aether_data_contracts::repository::routing_profiles::{
|
use aether_data_contracts::repository::routing_profiles::{
|
||||||
CreateRoutingGroupBindingRecord, CreateRoutingGroupRecord, RoutingGroupBindingSubject,
|
CreateRoutingGroupBindingRecord, CreateRoutingGroupRecord, RoutingGroupBindingSubject,
|
||||||
RoutingGroupWriteRepository,
|
RoutingGroupWriteRepository, UpdateRoutingGroupRecord,
|
||||||
};
|
};
|
||||||
use aether_provider_transport::snapshot::{
|
use aether_provider_transport::snapshot::{
|
||||||
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
|
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
|
||||||
@@ -1119,12 +1226,14 @@ mod tests {
|
|||||||
fn explicit_routing_selection_cache_key_is_principal_specific() {
|
fn explicit_routing_selection_cache_key_is_principal_specific() {
|
||||||
let first = routing_group_selection_cache_key(
|
let first = routing_group_selection_cache_key(
|
||||||
Some("private"),
|
Some("private"),
|
||||||
|
None,
|
||||||
Some("user-1"),
|
Some("user-1"),
|
||||||
Some("key-1"),
|
Some("key-1"),
|
||||||
&["team-1".to_string()],
|
&["team-1".to_string()],
|
||||||
);
|
);
|
||||||
let second = routing_group_selection_cache_key(
|
let second = routing_group_selection_cache_key(
|
||||||
Some("private"),
|
Some("private"),
|
||||||
|
None,
|
||||||
Some("user-2"),
|
Some("user-2"),
|
||||||
Some("key-2"),
|
Some("key-2"),
|
||||||
&["team-2".to_string()],
|
&["team-2".to_string()],
|
||||||
@@ -1151,6 +1260,50 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||||
|
let secret = "private-group?token=Bearer-secret";
|
||||||
|
|
||||||
|
for error in [
|
||||||
|
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||||
|
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||||
|
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||||
|
] {
|
||||||
|
let error = routing_selection_error(error);
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
ref message,
|
||||||
|
} if !message.contains(secret)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||||
|
let secret = "https://internal.example/?token=Bearer-secret";
|
||||||
|
let contract_error = invalid_routing_provider_contract();
|
||||||
|
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||||
|
format!("Authorization: {secret}"),
|
||||||
|
secret.to_string(),
|
||||||
|
)]))
|
||||||
|
.expect_err("invalid header should fail");
|
||||||
|
|
||||||
|
for (error, expected_message) in [
|
||||||
|
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||||
|
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
ref message,
|
||||||
|
} if message == expected_message && !message.contains(secret)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||||
// AppState::new has no auth snapshot repository. Without the explicit
|
// AppState::new has no auth snapshot repository. Without the explicit
|
||||||
@@ -1197,6 +1350,160 @@ mod tests {
|
|||||||
assert!(matches!(error, GatewayError::Internal(message) if message.contains("identity")));
|
assert!(matches!(error, GatewayError::Internal(message) if message.contains("identity")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn api_key_routing_selection_applies_at_planner_and_invalidates_after_changes() {
|
||||||
|
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(
|
||||||
|
["api-key-1", "api-key-2"].map(|key_id| {
|
||||||
|
(
|
||||||
|
None,
|
||||||
|
StoredAuthApiKeySnapshot::new(
|
||||||
|
"user-1".into(),
|
||||||
|
"alice".into(),
|
||||||
|
None,
|
||||||
|
"user".into(),
|
||||||
|
"local".into(),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
key_id.into(),
|
||||||
|
Some(key_id.into()),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
let groups = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||||
|
for (id, visible, is_default, multiplier) in [
|
||||||
|
("default", false, true, 1.0),
|
||||||
|
("discount", true, false, 0.5),
|
||||||
|
("premium", true, false, 2.0),
|
||||||
|
] {
|
||||||
|
groups.create_routing_group(CreateRoutingGroupRecord {
|
||||||
|
id: id.into(), name: format!("{id}-name"), description: None,
|
||||||
|
enabled: true, is_system_default: is_default, sort_order: 0,
|
||||||
|
config_json: json!({ "user_visible": visible, "billing_multiplier": multiplier }),
|
||||||
|
version: 1, created_at: 1, updated_at: 1, published_at: None,
|
||||||
|
}).await.unwrap();
|
||||||
|
}
|
||||||
|
let state = AppState::new().unwrap().with_data_state_for_tests(
|
||||||
|
crate::data::GatewayDataState::with_auth_api_key_repository_for_tests(auth_repository)
|
||||||
|
.with_routing_group_repository_for_tests(groups.clone()),
|
||||||
|
);
|
||||||
|
for (key_id, group_id) in [("api-key-1", "discount"), ("api-key-2", "premium")] {
|
||||||
|
assert!(state
|
||||||
|
.set_user_api_key_feature_settings(
|
||||||
|
"user-1",
|
||||||
|
key_id,
|
||||||
|
Some(json!({ "routing_group_id": group_id }))
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_some());
|
||||||
|
}
|
||||||
|
let (parts, _) = http::Request::builder().body(()).unwrap().into_parts();
|
||||||
|
let (header_parts, _) = http::Request::builder()
|
||||||
|
.header(ROUTING_GROUP_HEADER, "premium")
|
||||||
|
.body(())
|
||||||
|
.unwrap()
|
||||||
|
.into_parts();
|
||||||
|
|
||||||
|
async fn attach(
|
||||||
|
state: &AppState,
|
||||||
|
parts: &http::request::Parts,
|
||||||
|
key_id: &str,
|
||||||
|
) -> Result<LocalRequestedModelDecisionInput, GatewayError> {
|
||||||
|
let mut input = sample_decision_input();
|
||||||
|
input.auth_context.api_key_id = key_id.into();
|
||||||
|
input.auth_snapshot.api_key_id = key_id.into();
|
||||||
|
attach_routing_policy_to_local_requested_model_input(
|
||||||
|
state,
|
||||||
|
parts,
|
||||||
|
&mut input,
|
||||||
|
&json!({ "model": "gpt-5" }),
|
||||||
|
"openai:chat",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Revisit the first key after the second to exercise both cached choices.
|
||||||
|
for (key_id, group_id, multiplier) in [
|
||||||
|
("api-key-1", "discount", 0.5),
|
||||||
|
("api-key-2", "premium", 2.0),
|
||||||
|
("api-key-1", "discount", 0.5),
|
||||||
|
] {
|
||||||
|
let input = attach(&state, &parts, key_id).await.unwrap();
|
||||||
|
let policy = input.routing_policy.as_ref().unwrap();
|
||||||
|
assert_eq!(policy.group_id.as_deref(), Some(group_id));
|
||||||
|
assert_eq!(policy.selection_source, "api_key_selection");
|
||||||
|
assert_eq!(policy.billing_multiplier, multiplier);
|
||||||
|
assert_eq!(
|
||||||
|
input
|
||||||
|
.routing_trace_seed
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.billing_multiplier,
|
||||||
|
Some(multiplier)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||||
|
let policy = header.routing_policy.unwrap();
|
||||||
|
assert_eq!(policy.group_id.as_deref(), Some("premium"));
|
||||||
|
assert_eq!(policy.selection_source, "explicit_header");
|
||||||
|
|
||||||
|
groups
|
||||||
|
.update_routing_group(
|
||||||
|
"discount",
|
||||||
|
UpdateRoutingGroupRecord {
|
||||||
|
config_json: Some(json!({ "user_visible": false, "billing_multiplier": 0.5 })),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
state.invalidate_provider_routing_caches();
|
||||||
|
assert!(matches!(
|
||||||
|
attach(&state, &parts, "api-key-1").await,
|
||||||
|
Err(GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
header.routing_policy.unwrap().group_id.as_deref(),
|
||||||
|
Some("premium")
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(state
|
||||||
|
.set_user_api_key_feature_settings("user-1", "api-key-1", None)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.is_some());
|
||||||
|
let cleared = attach(&state, &parts, "api-key-1").await.unwrap();
|
||||||
|
let policy = cleared.routing_policy.unwrap();
|
||||||
|
assert_eq!(policy.group_id.as_deref(), Some("default"));
|
||||||
|
assert_eq!(policy.selection_source, "system_default");
|
||||||
|
assert_eq!(policy.billing_multiplier, 1.0);
|
||||||
|
// Clearing one key's preference must not disturb the other key's selection.
|
||||||
|
let other = attach(&state, &parts, "api-key-2").await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
other.routing_policy.unwrap().group_id.as_deref(),
|
||||||
|
Some("premium")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn explicit_routing_attachment_authorizes_and_caches_per_principal() {
|
async fn explicit_routing_attachment_authorizes_and_caches_per_principal() {
|
||||||
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
|
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||||
@@ -1208,7 +1515,7 @@ mod tests {
|
|||||||
enabled: true,
|
enabled: true,
|
||||||
is_system_default: false,
|
is_system_default: false,
|
||||||
sort_order: 0,
|
sort_order: 0,
|
||||||
config_json: json!({}),
|
config_json: json!({"billing_multiplier": 0.5}),
|
||||||
version: 1,
|
version: 1,
|
||||||
created_at: 1,
|
created_at: 1,
|
||||||
updated_at: 1,
|
updated_at: 1,
|
||||||
@@ -1254,6 +1561,11 @@ mod tests {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.expect("explicit selection should attach routing policy");
|
.expect("explicit selection should attach routing policy");
|
||||||
assert_eq!(policy.group_id.as_deref(), Some("private-group"));
|
assert_eq!(policy.group_id.as_deref(), Some("private-group"));
|
||||||
|
assert_eq!(policy.group_name.as_deref(), Some("private"));
|
||||||
|
assert_eq!(policy.billing_multiplier, 0.5);
|
||||||
|
let trace = allowed.routing_trace_seed.as_ref().unwrap();
|
||||||
|
assert_eq!(trace.group_name.as_deref(), Some("private"));
|
||||||
|
assert_eq!(trace.billing_multiplier, Some(0.5));
|
||||||
assert_eq!(policy.selection_source, "explicit_header");
|
assert_eq!(policy.selection_source, "explicit_header");
|
||||||
|
|
||||||
let mut denied = sample_decision_input();
|
let mut denied = sample_decision_input();
|
||||||
@@ -1328,7 +1640,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
model_directive_policy: Default::default(),
|
model_directive_policy: Default::default(),
|
||||||
@@ -1577,7 +1889,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
model_directive_policy: Default::default(),
|
model_directive_policy: Default::default(),
|
||||||
@@ -1621,7 +1933,7 @@ mod tests {
|
|||||||
assert_eq!(policy.group_version, Some(4));
|
assert_eq!(policy.group_version, Some(4));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
policy.priority_mode,
|
policy.priority_mode,
|
||||||
aether_routing_core::RoutingSetPriorityMode::GlobalKey
|
aether_routing_core::RoutingSetPriorityMode::Provider
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
policy.scheduling_mode,
|
policy.scheduling_mode,
|
||||||
@@ -1647,7 +1959,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -1716,6 +2028,35 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_codex_provider_outbound_policies_are_terminal_noop() {
|
||||||
|
let mut input = sample_decision_input();
|
||||||
|
input.routing_context = None;
|
||||||
|
input.provider_outbound_context = Some(ProviderOutboundRequestContext::new(
|
||||||
|
"logical-turn",
|
||||||
|
1_700_000_000_123,
|
||||||
|
));
|
||||||
|
let mut decision = sample_codex_fingerprint_decision();
|
||||||
|
decision.provider_type = Some("openai".to_string());
|
||||||
|
decision.provider_api_format = Some("openai:responses".to_string());
|
||||||
|
decision.client_api_format = Some("openai:responses".to_string());
|
||||||
|
let mut transport = sample_codex_fingerprint_transport();
|
||||||
|
transport.provider.provider_type = "openai".to_string();
|
||||||
|
|
||||||
|
let original_headers = decision.provider_request_headers.clone();
|
||||||
|
let original_body = decision.provider_request_body.clone();
|
||||||
|
apply_provider_request_routing_policy_to_decision(&input, &mut decision, Some(&transport))
|
||||||
|
.expect("non-Codex terminal finalization should succeed");
|
||||||
|
|
||||||
|
assert_eq!(decision.provider_request_headers, original_headers);
|
||||||
|
assert_eq!(decision.provider_request_body, original_body);
|
||||||
|
assert!(decision
|
||||||
|
.report_context
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|context| context.get("provider_outbound_policies"))
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
||||||
let transport = sample_codex_fingerprint_transport();
|
let transport = sample_codex_fingerprint_transport();
|
||||||
@@ -1732,8 +2073,8 @@ mod tests {
|
|||||||
});
|
});
|
||||||
let mut with_mutation = sample_decision_input();
|
let mut with_mutation = sample_decision_input();
|
||||||
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
||||||
input.codex_fingerprint_context = Some(
|
input.provider_outbound_context = Some(
|
||||||
CodexFingerprintConvergenceContext::new(
|
ProviderOutboundRequestContext::new(
|
||||||
uuid::Uuid::new_v4().to_string(),
|
uuid::Uuid::new_v4().to_string(),
|
||||||
1_756_668_000_000,
|
1_756_668_000_000,
|
||||||
)
|
)
|
||||||
@@ -1802,6 +2143,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
||||||
|
|
||||||
|
let policy_results = decision
|
||||||
|
.report_context
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|context| context.get("provider_outbound_policies"))
|
||||||
|
.and_then(|policies| policies.get("results"))
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.expect("provider policy results");
|
||||||
|
assert_eq!(
|
||||||
|
policy_results.len(),
|
||||||
|
1,
|
||||||
|
"policy result count at {exit_name}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
policy_results[0]["policy"],
|
||||||
|
json!("codex_fingerprint_convergence")
|
||||||
|
);
|
||||||
|
assert_eq!(policy_results[0]["outcome"], json!("applied"));
|
||||||
|
assert_eq!(policy_results[0]["reason"], json!("applied"));
|
||||||
|
|
||||||
let header_metadata: Value =
|
let header_metadata: Value =
|
||||||
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
||||||
.expect("header turn metadata");
|
.expect("header turn metadata");
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ use crate::ai_serving::transport::{
|
|||||||
build_gemini_cli_v1internal_request, build_standard_provider_request_headers,
|
build_gemini_cli_v1internal_request, build_standard_provider_request_headers,
|
||||||
GatewayProviderTransportSnapshot, GeminiCliRequestAuth, GeminiCliRequestAuthSupport,
|
GatewayProviderTransportSnapshot, GeminiCliRequestAuth, GeminiCliRequestAuthSupport,
|
||||||
GeminiCliRequestEnvelopeSupport, StandardProviderRequestHeaders,
|
GeminiCliRequestEnvelopeSupport, StandardProviderRequestHeaders,
|
||||||
StandardProviderRequestHeadersInput, GEMINI_CLI_USER_AGENT,
|
StandardProviderRequestHeadersInput,
|
||||||
};
|
};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
@@ -64,8 +64,10 @@ pub(crate) async fn build_gemini_cli_v1internal_provider_request(
|
|||||||
)
|
)
|
||||||
.ok_or(GeminiCliV1InternalRequestError::UpstreamUrlUnavailable)?;
|
.ok_or(GeminiCliV1InternalRequestError::UpstreamUrlUnavailable)?;
|
||||||
|
|
||||||
let extra_headers =
|
let extra_headers = BTreeMap::from([(
|
||||||
BTreeMap::from([("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string())]);
|
"user-agent".to_string(),
|
||||||
|
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||||
|
)]);
|
||||||
let headers = build_standard_provider_request_headers(StandardProviderRequestHeadersInput {
|
let headers = build_standard_provider_request_headers(StandardProviderRequestHeadersInput {
|
||||||
transport: &payload.transport,
|
transport: &payload.transport,
|
||||||
provider_api_format: input.provider_api_format,
|
provider_api_format: input.provider_api_format,
|
||||||
|
|||||||
@@ -21,10 +21,11 @@ use crate::ai_serving::transport::{
|
|||||||
build_same_format_provider_headers, resolve_local_gemini_cli_request_auth,
|
build_same_format_provider_headers, resolve_local_gemini_cli_request_auth,
|
||||||
GeminiCliRequestAuth, GeminiCliRequestAuthSupport, GeminiCliRequestEnvelopeSupport,
|
GeminiCliRequestAuth, GeminiCliRequestAuthSupport, GeminiCliRequestEnvelopeSupport,
|
||||||
GrokHeaderInput, SameFormatProviderCompatibilityEdit,
|
GrokHeaderInput, SameFormatProviderCompatibilityEdit,
|
||||||
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput,
|
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput, GROK_CHAT_PATH,
|
||||||
GEMINI_CLI_USER_AGENT, GROK_CHAT_PATH,
|
};
|
||||||
|
use crate::ai_serving::{
|
||||||
|
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, CODEX_RESPONSES_LITE_HEADER,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
mod policy;
|
mod policy;
|
||||||
@@ -255,7 +256,9 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
|||||||
// re-enforce stream-field policy afterward.
|
// re-enforce stream-field policy afterward.
|
||||||
// Kiro behavior classification already hard-requires upstream streaming,
|
// Kiro behavior classification already hard-requires upstream streaming,
|
||||||
// and the Kiro envelope does not use a top-level body stream field.
|
// and the Kiro envelope does not use a top-level body stream field.
|
||||||
if prepared.kiro_auth.is_none() {
|
if prepared.kiro_auth.is_none()
|
||||||
|
&& spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize)
|
||||||
|
{
|
||||||
enforce_provider_body_stream_policy(
|
enforce_provider_body_stream_policy(
|
||||||
&mut base_provider_request_body,
|
&mut base_provider_request_body,
|
||||||
prepared.provider_api_format.as_str(),
|
prepared.provider_api_format.as_str(),
|
||||||
@@ -275,7 +278,8 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
|||||||
prepared.mapped_model.as_str(),
|
prepared.mapped_model.as_str(),
|
||||||
source_model,
|
source_model,
|
||||||
);
|
);
|
||||||
if let Err(violation) =
|
if spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||||
|
if let Err(violation) =
|
||||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||||
&mut base_provider_request_body,
|
&mut base_provider_request_body,
|
||||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||||
@@ -313,6 +317,21 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
|||||||
.await;
|
.await;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same-format requests skip `apply_transport_request_body_semantics`, so the opt-in
|
||||||
|
// Claude Code body mimicry has to be applied here as well.
|
||||||
|
if crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||||
|
&mut base_provider_request_body,
|
||||||
|
&transport,
|
||||||
|
prepared.provider_api_format.as_str(),
|
||||||
|
) {
|
||||||
|
compatibility_edits.push(SameFormatProviderCompatibilityEdit {
|
||||||
|
field: "body".to_string(),
|
||||||
|
action: SameFormatProviderCompatibilityEditAction::ProviderCompatibilityRewrite,
|
||||||
|
detail: "applied Claude Code body mimicry for provider compatibility".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let antigravity_auth = if prepared.is_antigravity {
|
let antigravity_auth = if prepared.is_antigravity {
|
||||||
let mut antigravity_support = classify_local_antigravity_request_support(
|
let mut antigravity_support = classify_local_antigravity_request_support(
|
||||||
@@ -513,7 +532,10 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
|||||||
.map(build_antigravity_static_identity_headers)
|
.map(build_antigravity_static_identity_headers)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
if prepared.behavior.is_gemini_cli {
|
if prepared.behavior.is_gemini_cli {
|
||||||
extra_headers.insert("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string());
|
extra_headers.insert(
|
||||||
|
"user-agent".to_string(),
|
||||||
|
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let Some(mut provider_request_headers) = (if is_grok {
|
let Some(mut provider_request_headers) = (if is_grok {
|
||||||
build_grok_browser_headers(GrokHeaderInput {
|
build_grok_browser_headers(GrokHeaderInput {
|
||||||
@@ -583,6 +605,16 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
|||||||
source_model,
|
source_model,
|
||||||
codex_model_capabilities.as_ref(),
|
codex_model_capabilities.as_ref(),
|
||||||
);
|
);
|
||||||
|
if spec.operation == Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||||
|
provider_request_headers
|
||||||
|
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||||
|
provider_request_headers.insert("accept".to_string(), "application/json".to_string());
|
||||||
|
}
|
||||||
|
crate::ai_serving::transport::xai::insert_cli_identity_headers_if_needed(
|
||||||
|
transport.as_ref(),
|
||||||
|
prepared.provider_api_format.as_str(),
|
||||||
|
&mut provider_request_headers,
|
||||||
|
);
|
||||||
request_identity_response_encoding_when_redacted(
|
request_identity_response_encoding_when_redacted(
|
||||||
&mut provider_request_headers,
|
&mut provider_request_headers,
|
||||||
redaction.redacted,
|
redaction.redacted,
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncA
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
@@ -246,7 +246,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt>
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
|||||||
use super::super::LocalSameFormatProviderSpec;
|
use super::super::LocalSameFormatProviderSpec;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_same_format_provider_request_body as build_same_format_provider_request_body_impl,
|
build_same_format_provider_request_body as build_same_format_provider_request_body_impl,
|
||||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy as build_same_format_provider_request_body_with_compatibility_report_impl,
|
build_same_format_provider_request_body_for_operation as build_same_format_provider_request_body_with_compatibility_report_impl,
|
||||||
SameFormatProviderFamily, SameFormatProviderRequestBodyInput,
|
SameFormatProviderFamily, SameFormatProviderRequestBodyInput,
|
||||||
SameFormatProviderRequestBodyOutput,
|
SameFormatProviderRequestBodyOutput,
|
||||||
};
|
};
|
||||||
@@ -69,6 +69,7 @@ pub(crate) fn build_same_format_provider_request_body_with_compatibility_report(
|
|||||||
enable_model_directives,
|
enable_model_directives,
|
||||||
},
|
},
|
||||||
reasoning_replay_policy,
|
reasoning_replay_policy,
|
||||||
|
spec.operation,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,11 @@ use aether_ai_serving::{
|
|||||||
provider_stream_event_api_format_for_provider_type as ai_provider_stream_event_api_format_for_provider_type,
|
provider_stream_event_api_format_for_provider_type as ai_provider_stream_event_api_format_for_provider_type,
|
||||||
AiExecutionReportContextParts, AiRequestOrigin, STICKY_KEY_ATTEMPTS_REPORT_FIELD,
|
AiExecutionReportContextParts, AiRequestOrigin, STICKY_KEY_ATTEMPTS_REPORT_FIELD,
|
||||||
};
|
};
|
||||||
|
use aether_data_contracts::repository::usage::{
|
||||||
|
BillingMultiplierSnapshot, BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY,
|
||||||
|
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY, ROUTING_GROUP_ID_METADATA_KEY,
|
||||||
|
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||||
|
};
|
||||||
use aether_routing_core::ResolvedRoutingPolicy;
|
use aether_routing_core::ResolvedRoutingPolicy;
|
||||||
use aether_runtime_state::RuntimeLockLease;
|
use aether_runtime_state::RuntimeLockLease;
|
||||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
||||||
@@ -76,15 +81,57 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
let RequestOrigin {
|
let RequestOrigin {
|
||||||
client_ip,
|
client_ip,
|
||||||
user_agent,
|
user_agent,
|
||||||
|
forwarded_headers_trusted,
|
||||||
} = parts
|
} = parts
|
||||||
.request_origin
|
.request_origin
|
||||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
let original_headers =
|
||||||
|
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||||
parts.original_request_body_json,
|
parts.original_request_body_json,
|
||||||
parts.original_request_body_base64,
|
parts.original_request_body_base64,
|
||||||
);
|
);
|
||||||
let mut extra_fields = parts.extra_fields;
|
let mut extra_fields = parts.extra_fields;
|
||||||
|
// Always overwrite caller-supplied extras with the planner's immutable policy snapshot.
|
||||||
|
let billing_multiplier = parts
|
||||||
|
.routing_policy
|
||||||
|
.map(|policy| policy.billing_multiplier)
|
||||||
|
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||||
|
.unwrap_or(1.0);
|
||||||
|
extra_fields.insert(
|
||||||
|
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY.to_string(),
|
||||||
|
Value::from(billing_multiplier),
|
||||||
|
);
|
||||||
|
extra_fields.insert(
|
||||||
|
BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY.to_string(),
|
||||||
|
serde_json::to_value(
|
||||||
|
BillingMultiplierSnapshot::from_factors(BTreeMap::from([(
|
||||||
|
"routing_group".to_string(),
|
||||||
|
billing_multiplier,
|
||||||
|
)]))
|
||||||
|
.expect("validated routing multiplier must produce a billing snapshot"),
|
||||||
|
)
|
||||||
|
.expect("validated billing snapshot must serialize"),
|
||||||
|
);
|
||||||
|
for (field, value) in [
|
||||||
|
(
|
||||||
|
ROUTING_GROUP_ID_METADATA_KEY,
|
||||||
|
parts
|
||||||
|
.routing_policy
|
||||||
|
.and_then(|policy| policy.group_id.as_deref()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||||
|
parts
|
||||||
|
.routing_policy
|
||||||
|
.and_then(|policy| policy.group_name.as_deref()),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
extra_fields.remove(field);
|
||||||
|
if let Some(value) = value {
|
||||||
|
extra_fields.insert(field.to_string(), Value::String(value.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Some(value) = parts
|
if let Some(value) = parts
|
||||||
.client_session_affinity
|
.client_session_affinity
|
||||||
.and_then(client_session_affinity_report_context_value)
|
.and_then(client_session_affinity_report_context_value)
|
||||||
@@ -106,15 +153,17 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
value,
|
value,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if let Some(incoming_tls) =
|
if forwarded_headers_trusted {
|
||||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
if let Some(incoming_tls) =
|
||||||
{
|
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
{
|
||||||
|
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||||
if let Some(policy) = parts.routing_policy {
|
if let Some(policy) = parts.routing_policy {
|
||||||
if let Ok(value) = serde_json::to_value(policy.execution_policy) {
|
if let Ok(value) = serde_json::to_value(&policy.execution_policy) {
|
||||||
extra_fields.insert(ROUTING_EXECUTION_POLICY_REPORT_FIELD.to_string(), value);
|
extra_fields.insert(ROUTING_EXECUTION_POLICY_REPORT_FIELD.to_string(), value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -189,6 +238,17 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn collect_report_context_original_headers(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
forwarded_headers_trusted: bool,
|
||||||
|
) -> BTreeMap<String, String> {
|
||||||
|
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||||
|
if !forwarded_headers_trusted {
|
||||||
|
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||||
|
}
|
||||||
|
collected
|
||||||
|
}
|
||||||
|
|
||||||
fn insert_request_path_fields(
|
fn insert_request_path_fields(
|
||||||
extra_fields: &mut Map<String, Value>,
|
extra_fields: &mut Map<String, Value>,
|
||||||
request_path: Option<&str>,
|
request_path: Option<&str>,
|
||||||
@@ -258,8 +318,8 @@ mod tests {
|
|||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
build_local_execution_report_context, collect_report_context_original_headers,
|
||||||
LocalExecutionReportContextParts,
|
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||||
use crate::ai_serving::RequestOrigin;
|
use crate::ai_serving::RequestOrigin;
|
||||||
@@ -289,6 +349,26 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||||
|
let mut headers = http::HeaderMap::new();
|
||||||
|
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||||
|
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||||
|
|
||||||
|
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||||
|
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||||
|
assert_eq!(
|
||||||
|
untrusted.get("user-agent").map(String::as_str),
|
||||||
|
Some("test-client")
|
||||||
|
);
|
||||||
|
|
||||||
|
let trusted = collect_report_context_original_headers(&headers, true);
|
||||||
|
assert_eq!(
|
||||||
|
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||||
|
Some("spoofed-ja3")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||||
let auth_context = ExecutionRuntimeAuthContext {
|
let auth_context = ExecutionRuntimeAuthContext {
|
||||||
@@ -306,6 +386,27 @@ mod tests {
|
|||||||
Some("codex".to_string()),
|
Some("codex".to_string()),
|
||||||
Some("account=account-1;session=session-1".to_string()),
|
Some("account=account-1;session=session-1".to_string()),
|
||||||
);
|
);
|
||||||
|
let mut routing_policy = aether_routing_core::resolve_routing_policy(
|
||||||
|
&aether_routing_core::RoutingGroupConfig {
|
||||||
|
billing_multiplier: 0.25,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
aether_routing_core::RoutingPolicyInput {
|
||||||
|
group_id: Some("group-1"),
|
||||||
|
group_version: Some(7),
|
||||||
|
selection_source: "system_default",
|
||||||
|
requested_model: "gpt-5",
|
||||||
|
resolved_model: "gpt-5",
|
||||||
|
api_format: "openai:chat",
|
||||||
|
user_id: None,
|
||||||
|
api_key_id: None,
|
||||||
|
headers: &json!({}),
|
||||||
|
body: &json!({}),
|
||||||
|
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("routing policy should resolve");
|
||||||
|
routing_policy.group_name = Some("请求时的分组".to_string());
|
||||||
|
|
||||||
let report_context =
|
let report_context =
|
||||||
build_local_execution_report_context(LocalExecutionReportContextParts {
|
build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||||
@@ -339,20 +440,40 @@ mod tests {
|
|||||||
request_origin: Some(RequestOrigin {
|
request_origin: Some(RequestOrigin {
|
||||||
client_ip: Some("203.0.113.8".to_string()),
|
client_ip: Some("203.0.113.8".to_string()),
|
||||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||||
|
forwarded_headers_trusted: false,
|
||||||
}),
|
}),
|
||||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
client_session_affinity: Some(&client_session_affinity),
|
client_session_affinity: Some(&client_session_affinity),
|
||||||
routing_policy: None,
|
routing_policy: Some(&routing_policy),
|
||||||
scheduler_affinity_epoch: None,
|
scheduler_affinity_epoch: None,
|
||||||
sticky_key_attempts: None,
|
sticky_key_attempts: None,
|
||||||
client_requested_stream: false,
|
client_requested_stream: false,
|
||||||
upstream_is_stream: false,
|
upstream_is_stream: false,
|
||||||
has_envelope: false,
|
has_envelope: false,
|
||||||
needs_conversion: false,
|
needs_conversion: false,
|
||||||
extra_fields: Map::new(),
|
extra_fields: Map::from_iter([
|
||||||
|
(
|
||||||
|
"billing_multiplier_snapshot".to_string(),
|
||||||
|
json!({
|
||||||
|
"version": 1, "factors": {"routing_group": 99.0}, "multiplier": 99.0
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
("routing_group_billing_multiplier".to_string(), json!(99)),
|
||||||
|
("routing_group_id".to_string(), json!("forged-group")),
|
||||||
|
("routing_group_name".to_string(), json!("forged-name")),
|
||||||
|
]),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
assert_eq!(report_context["routing_group_billing_multiplier"], 0.25);
|
||||||
|
assert_eq!(
|
||||||
|
report_context["billing_multiplier_snapshot"],
|
||||||
|
json!({
|
||||||
|
"version": 1, "factors": {"routing_group": 0.25}, "multiplier": 0.25
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(report_context["routing_group_id"], "group-1");
|
||||||
|
assert_eq!(report_context["routing_group_name"], "请求时的分组");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report_context["client_ip"],
|
report_context["client_ip"],
|
||||||
Value::String("203.0.113.8".to_string())
|
Value::String("203.0.113.8".to_string())
|
||||||
@@ -491,7 +612,11 @@ mod tests {
|
|||||||
original_headers: &original_headers,
|
original_headers: &original_headers,
|
||||||
request_path: None,
|
request_path: None,
|
||||||
request_query_string: None,
|
request_query_string: None,
|
||||||
request_origin: None,
|
request_origin: Some(RequestOrigin {
|
||||||
|
client_ip: None,
|
||||||
|
user_agent: None,
|
||||||
|
forwarded_headers_trusted: true,
|
||||||
|
}),
|
||||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptS
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
@@ -224,7 +224,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAtte
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
|||||||
GeminiFilesRequestBodyError,
|
GeminiFilesRequestBodyError,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
use crate::ai_serving::{
|
||||||
|
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||||
|
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||||
|
};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
use super::support::{
|
use super::support::{
|
||||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
|||||||
let transport = &attempt.eligible.transport;
|
let transport = &attempt.eligible.transport;
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
|
||||||
|
if matches!(
|
||||||
|
spec_metadata.decision_kind,
|
||||||
|
GEMINI_FILES_GET_PLAN_KIND
|
||||||
|
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||||
|
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||||
|
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||||
|
{
|
||||||
|
mark_skipped_local_gemini_files_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"gemini_file_mapping_mismatch",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(skip_reason) =
|
if let Some(skip_reason) =
|
||||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||||
{
|
{
|
||||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
|||||||
file_name,
|
file_name,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn candidate_matches_owned_gemini_file_mapping(
|
||||||
|
state: &AppState,
|
||||||
|
parts: &http::request::Parts,
|
||||||
|
input: &LocalGeminiFilesDecisionInput,
|
||||||
|
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||||
|
) -> bool {
|
||||||
|
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let user_id = input.auth_context.user_id.trim();
|
||||||
|
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Ok(Some(mapping)) = state
|
||||||
|
.find_active_gemini_file_mapping_for_owner(
|
||||||
|
file_name.as_str(),
|
||||||
|
&attempt.eligible.transport.key.id,
|
||||||
|
user_id,
|
||||||
|
crate::clock::current_unix_secs(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||||
|
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||||
|
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||||
|
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||||
|
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||||
|
if suffix.is_empty() || suffix.contains('/') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(format!("files/{suffix}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::normalize_gemini_file_name_from_path;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizes_supported_gemini_file_object_paths() {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||||
|
Some("files/file-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||||
|
Some("files/file-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||||
|
Some("files/abc-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -257,7 +257,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptS
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
@@ -302,7 +302,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAtte
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::candidate_preparation::{
|
|||||||
};
|
};
|
||||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||||
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
||||||
|
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
||||||
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
||||||
@@ -16,8 +17,8 @@ use crate::ai_serving::transport::{
|
|||||||
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
|
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
apply_codex_openai_special_headers, build_chatgpt_web_image_request_body,
|
apply_codex_openai_special_headers, apply_xai_upstream_payload_edits,
|
||||||
build_codex_openai_image_api_provider_request_body,
|
build_chatgpt_web_image_request_body, build_codex_openai_image_api_provider_request_body,
|
||||||
build_gemini_image_request_body_from_openai_image_request,
|
build_gemini_image_request_body_from_openai_image_request,
|
||||||
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
|
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
|
||||||
default_model_for_openai_image_operation, normalize_openai_image_request,
|
default_model_for_openai_image_operation, normalize_openai_image_request,
|
||||||
@@ -210,7 +211,7 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
|||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let Some(provider_request_body) = provider_request_body else {
|
let Some(mut provider_request_body) = provider_request_body else {
|
||||||
mark_skipped_local_openai_image_candidate_with_failure_diagnostic(
|
mark_skipped_local_openai_image_candidate_with_failure_diagnostic(
|
||||||
state,
|
state,
|
||||||
input,
|
input,
|
||||||
@@ -228,6 +229,11 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
|||||||
.await;
|
.await;
|
||||||
return None;
|
return None;
|
||||||
};
|
};
|
||||||
|
apply_xai_upstream_payload_edits(
|
||||||
|
&mut provider_request_body,
|
||||||
|
transport.provider.provider_type.as_str(),
|
||||||
|
provider_api_format,
|
||||||
|
);
|
||||||
let Some(mut provider_request_headers) = (if is_grok {
|
let Some(mut provider_request_headers) = (if is_grok {
|
||||||
build_grok_browser_headers(GrokHeaderInput {
|
build_grok_browser_headers(GrokHeaderInput {
|
||||||
transport,
|
transport,
|
||||||
@@ -338,6 +344,25 @@ async fn resolve_local_openai_image_to_gemini_candidate_payload_parts(
|
|||||||
let candidate = &attempt.eligible.candidate;
|
let candidate = &attempt.eligible.candidate;
|
||||||
let transport = &attempt.eligible.transport;
|
let transport = &attempt.eligible.transport;
|
||||||
let provider_api_format = "gemini:generate_content";
|
let provider_api_format = "gemini:generate_content";
|
||||||
|
|
||||||
|
// The gemini:generate_content URL hook rewrites an Antigravity endpoint to
|
||||||
|
// /v1internal:, and this image path has no v1internal envelope to match it.
|
||||||
|
// Skip the candidate instead of posting a bare Gemini body that upstream
|
||||||
|
// would only reject.
|
||||||
|
if is_antigravity_provider_transport(transport) {
|
||||||
|
mark_skipped_local_openai_image_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_unsupported",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
|
||||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptS
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use crate::ai_serving::planner::{
|
|||||||
build_ai_execution_decision_response, resolve_transport_request_encoding_policy,
|
build_ai_execution_decision_response, resolve_transport_request_encoding_policy,
|
||||||
AiExecutionDecisionResponseParts,
|
AiExecutionDecisionResponseParts,
|
||||||
};
|
};
|
||||||
|
use crate::ai_serving::transport::xai::video::is_native_video_request;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
resolve_transport_execution_timeouts, resolve_transport_profile,
|
resolve_transport_execution_timeouts, resolve_transport_profile,
|
||||||
};
|
};
|
||||||
@@ -33,7 +34,7 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
|||||||
let Some(resolved) = resolve_local_video_create_candidate_payload_parts(
|
let Some(resolved) = resolve_local_video_create_candidate_payload_parts(
|
||||||
state, parts, body_json, trace_id, input, &attempt, spec,
|
state, parts, body_json, trace_id, input, &attempt, spec,
|
||||||
)
|
)
|
||||||
.await
|
.await?
|
||||||
else {
|
else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
@@ -52,9 +53,32 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
|||||||
.await;
|
.await;
|
||||||
let transport_profile = resolve_transport_profile(&transport);
|
let transport_profile = resolve_transport_profile(&transport);
|
||||||
let mut extra_fields = serde_json::Map::new();
|
let mut extra_fields = serde_json::Map::new();
|
||||||
|
if is_native_video_request(&transport.provider.provider_type, parts.uri.path()) {
|
||||||
|
extra_fields.insert(
|
||||||
|
"video_client_protocol".to_string(),
|
||||||
|
serde_json::json!("xai"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(proxy_value) = build_request_trace_proxy_value(Some(&transport), proxy.as_ref()) {
|
if let Some(proxy_value) = build_request_trace_proxy_value(Some(&transport), proxy.as_ref()) {
|
||||||
extra_fields.insert("proxy".to_string(), proxy_value);
|
extra_fields.insert("proxy".to_string(), proxy_value);
|
||||||
}
|
}
|
||||||
|
if transport.provider.provider_type.eq_ignore_ascii_case("xai") {
|
||||||
|
extra_fields.insert("video_provider_xai".into(), serde_json::json!(true));
|
||||||
|
if let Some(duration) = resolved.provider_request_body.get("duration") {
|
||||||
|
extra_fields.insert("video_duration".into(), duration.clone());
|
||||||
|
}
|
||||||
|
if parts.uri.path() == "/openai/v1/videos" {
|
||||||
|
extra_fields.insert(
|
||||||
|
"video_size".into(),
|
||||||
|
body_json
|
||||||
|
.get("size")
|
||||||
|
.filter(|v| v.as_str().is_some_and(|s| !s.trim().is_empty()))
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| serde_json::json!("720x1280")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
|
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||||
auth_context: &input.auth_context,
|
auth_context: &input.auth_context,
|
||||||
|
|||||||
@@ -3,15 +3,23 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::ai_serving::planner::candidate_preparation::resolve_candidate_mapped_model;
|
use crate::ai_serving::planner::candidate_preparation::{
|
||||||
|
prepare_header_authenticated_candidate, resolve_candidate_mapped_model, OauthPreparationContext,
|
||||||
|
};
|
||||||
use crate::ai_serving::planner::spec_metadata::local_video_create_spec_metadata;
|
use crate::ai_serving::planner::spec_metadata::local_video_create_spec_metadata;
|
||||||
|
use crate::ai_serving::transport::xai::video::{
|
||||||
|
convert_openai_video_request, is_explicit_native_video_path, is_native_video_request,
|
||||||
|
};
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
|
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
|
||||||
resolve_video_create_auth, video_create_transport_unsupported_reason,
|
resolve_video_create_auth, video_create_transport_unsupported_reason,
|
||||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
use crate::ai_serving::{
|
||||||
use crate::AppState;
|
apply_xai_upstream_payload_edits, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
|
||||||
|
PlannerAppState,
|
||||||
|
};
|
||||||
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
use super::support::{
|
use super::support::{
|
||||||
mark_skipped_local_video_candidate, mark_skipped_local_video_candidate_with_failure_diagnostic,
|
mark_skipped_local_video_candidate, mark_skipped_local_video_candidate_with_failure_diagnostic,
|
||||||
@@ -37,11 +45,16 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
input: &LocalVideoCreateDecisionInput,
|
input: &LocalVideoCreateDecisionInput,
|
||||||
attempt: &LocalVideoCreateCandidateAttempt,
|
attempt: &LocalVideoCreateCandidateAttempt,
|
||||||
spec: LocalVideoCreateSpec,
|
spec: LocalVideoCreateSpec,
|
||||||
) -> Option<LocalVideoCreateCandidatePayloadParts> {
|
) -> Result<Option<LocalVideoCreateCandidatePayloadParts>, GatewayError> {
|
||||||
let spec_metadata = local_video_create_spec_metadata(spec);
|
let spec_metadata = local_video_create_spec_metadata(spec);
|
||||||
let candidate = &attempt.eligible.candidate;
|
let candidate = &attempt.eligible.candidate;
|
||||||
let transport = &attempt.eligible.transport;
|
let transport = &attempt.eligible.transport;
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
if is_explicit_native_video_path(parts.uri.path())
|
||||||
|
&& !transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||||
|
{
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
let provider_family = provider_video_create_family(spec.family);
|
let provider_family = provider_video_create_family(spec.family);
|
||||||
let transport_unsupported_reason = video_create_transport_unsupported_reason(
|
let transport_unsupported_reason = video_create_transport_unsupported_reason(
|
||||||
@@ -60,23 +73,39 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
skip_reason,
|
skip_reason,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return None;
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let auth = resolve_video_create_auth(transport, provider_family);
|
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||||
let Some((auth_header, auth_value)) = auth else {
|
PlannerAppState::new(state),
|
||||||
mark_skipped_local_video_candidate(
|
transport,
|
||||||
state,
|
candidate,
|
||||||
input,
|
resolve_video_create_auth(transport, provider_family),
|
||||||
|
OauthPreparationContext {
|
||||||
trace_id,
|
trace_id,
|
||||||
candidate,
|
api_format: spec_metadata.api_format,
|
||||||
attempt.candidate_index,
|
operation: "video_create_candidate_request",
|
||||||
&attempt.candidate_id,
|
},
|
||||||
"transport_auth_unavailable",
|
)
|
||||||
)
|
.await
|
||||||
.await;
|
{
|
||||||
return None;
|
Ok(prepared) => prepared,
|
||||||
|
Err(skip_reason) => {
|
||||||
|
mark_skipped_local_video_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
skip_reason,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
let auth_header = prepared_candidate.auth_header;
|
||||||
|
let auth_value = prepared_candidate.auth_value;
|
||||||
|
|
||||||
let mapped_model = match resolve_candidate_mapped_model(candidate) {
|
let mapped_model = match resolve_candidate_mapped_model(candidate) {
|
||||||
Ok(mapped_model) => mapped_model,
|
Ok(mapped_model) => mapped_model,
|
||||||
@@ -91,7 +120,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
skip_reason,
|
skip_reason,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return None;
|
return Ok(None);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -117,10 +146,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return None;
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(provider_request_body) = build_video_create_request_body(
|
let Some(mut provider_request_body) = build_video_create_request_body(
|
||||||
body_json,
|
body_json,
|
||||||
provider_family,
|
provider_family,
|
||||||
&mapped_model,
|
&mapped_model,
|
||||||
@@ -142,11 +171,28 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return None;
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
if transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||||
|
&& !is_native_video_request(&transport.provider.provider_type, parts.uri.path())
|
||||||
|
{
|
||||||
|
provider_request_body =
|
||||||
|
convert_openai_video_request(&provider_request_body).map_err(|message| {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: http::StatusCode::BAD_REQUEST,
|
||||||
|
message: message.to_string(),
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
apply_xai_upstream_payload_edits(
|
||||||
|
&mut provider_request_body,
|
||||||
|
transport.provider.provider_type.as_str(),
|
||||||
|
spec_metadata.api_format,
|
||||||
|
);
|
||||||
|
|
||||||
let Some(provider_request_headers) =
|
let Some(provider_request_headers) =
|
||||||
build_video_create_headers(ProviderVideoCreateHeadersInput {
|
build_video_create_headers(ProviderVideoCreateHeadersInput {
|
||||||
|
transport,
|
||||||
headers: effective_headers,
|
headers: effective_headers,
|
||||||
auth_header: &auth_header,
|
auth_header: &auth_header,
|
||||||
auth_value: &auth_value,
|
auth_value: &auth_value,
|
||||||
@@ -170,10 +216,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return None;
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(LocalVideoCreateCandidatePayloadParts {
|
Ok(Some(LocalVideoCreateCandidatePayloadParts {
|
||||||
transport: Arc::clone(transport),
|
transport: Arc::clone(transport),
|
||||||
auth_header,
|
auth_header,
|
||||||
auth_value,
|
auth_value,
|
||||||
@@ -181,7 +227,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
|||||||
provider_request_headers,
|
provider_request_headers,
|
||||||
provider_request_body,
|
provider_request_body,
|
||||||
upstream_url,
|
upstream_url,
|
||||||
})
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn provider_video_create_family(family: LocalVideoCreateFamily) -> ProviderVideoCreateFamily {
|
fn provider_video_create_family(family: LocalVideoCreateFamily) -> ProviderVideoCreateFamily {
|
||||||
|
|||||||
@@ -504,10 +504,13 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
|||||||
);
|
);
|
||||||
assert_eq!(headers.get("x-client-request-id"), None);
|
assert_eq!(headers.get("x-client-request-id"), None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers.get("originator"),
|
||||||
|
Some(&aether_ai_formats::codex_client_originator())
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -614,10 +617,13 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
|||||||
Some(&"acc-123".to_string())
|
Some(&"acc-123".to_string())
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers.get("originator"),
|
||||||
|
Some(&aether_ai_formats::codex_client_originator())
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||||
for name in ["x-client-request-id", "session-id", "thread-id"] {
|
for name in ["x-client-request-id", "session-id", "thread-id"] {
|
||||||
@@ -698,10 +704,13 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
|||||||
Some(&"kept-by-rule-request".to_string())
|
Some(&"kept-by-rule-request".to_string())
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers.get("originator"),
|
||||||
|
Some(&aether_ai_formats::codex_client_originator())
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers
|
headers
|
||||||
.keys()
|
.keys()
|
||||||
@@ -762,10 +771,13 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
|||||||
);
|
);
|
||||||
assert_eq!(headers.get("x-client-request-id"), None);
|
assert_eq!(headers.get("x-client-request-id"), None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers.get("originator"),
|
||||||
|
Some(&aether_ai_formats::codex_client_originator())
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -1,39 +1,21 @@
|
|||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
pub(crate) fn is_deepseek_provider(provider_type: &str, base_url: &str) -> bool {
|
pub(crate) fn is_deepseek_provider(_provider_type: &str, base_url: &str) -> bool {
|
||||||
let provider_type = provider_type.trim().to_ascii_lowercase();
|
|
||||||
if matches!(
|
|
||||||
provider_type.as_str(),
|
|
||||||
"deepseek" | "deepseek_openai" | "deepseek_anthropic" | "deepseek_compatible"
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
let Some(host) = base_url_host(base_url) else {
|
let Some(host) = base_url_host(base_url) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
host == "deepseek.com" || host.ends_with(".deepseek.com")
|
// 仅官方接口启用专用兼容;供应商类型和模型名称不能代表第三方接口的行为。
|
||||||
}
|
host == "api.deepseek.com"
|
||||||
|
|
||||||
fn is_deepseek_model(provider_model: &str) -> bool {
|
|
||||||
let provider_model = provider_model.trim().to_ascii_lowercase();
|
|
||||||
let leaf = provider_model
|
|
||||||
.rsplit(['/', ':'])
|
|
||||||
.next()
|
|
||||||
.unwrap_or(provider_model.as_str());
|
|
||||||
leaf == "deepseek" || leaf.starts_with("deepseek-") || leaf.starts_with("deepseek_")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_deepseek_upstream(provider_type: &str, base_url: &str, provider_model: &str) -> bool {
|
|
||||||
is_deepseek_provider(provider_type, base_url) || is_deepseek_model(provider_model)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn openai_responses_reasoning_replay_policy(
|
pub(crate) fn openai_responses_reasoning_replay_policy(
|
||||||
provider_type: &str,
|
provider_type: &str,
|
||||||
base_url: &str,
|
base_url: &str,
|
||||||
provider_model: &str,
|
_provider_model: &str,
|
||||||
) -> crate::ai_serving::OpenAiResponsesReasoningReplayPolicy {
|
) -> crate::ai_serving::OpenAiResponsesReasoningReplayPolicy {
|
||||||
if is_deepseek_upstream(provider_type, base_url, provider_model) {
|
if provider_type.trim().eq_ignore_ascii_case("xai") {
|
||||||
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||||
|
} else if is_deepseek_provider(provider_type, base_url) {
|
||||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||||
} else {
|
} else {
|
||||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||||
@@ -47,11 +29,7 @@ pub(crate) fn apply_deepseek_tool_call_thinking_compat(
|
|||||||
provider_api_format: &str,
|
provider_api_format: &str,
|
||||||
original_request_body: Option<&Value>,
|
original_request_body: Option<&Value>,
|
||||||
) {
|
) {
|
||||||
let provider_model = provider_request_body
|
if !is_deepseek_provider(provider_type, base_url) {
|
||||||
.get("model")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.unwrap_or_default();
|
|
||||||
if !is_deepseek_upstream(provider_type, base_url, provider_model) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,6 +133,9 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
|||||||
provider_request_body: &mut Value,
|
provider_request_body: &mut Value,
|
||||||
original_request_body: Option<&Value>,
|
original_request_body: Option<&Value>,
|
||||||
) {
|
) {
|
||||||
|
// 携带 tools 时,所有历史 reasoning_content 都须完整回传,包括未调用工具的轮次。
|
||||||
|
// 无 tools 时允许回传,且 prefix 续写需要保留输入;因此原样保留 messages,
|
||||||
|
// 不删除思考内容,也不以空字符串冒充缺失内容,由上游校验请求是否完整。
|
||||||
let disabled = source_disables_thinking(original_request_body, provider_request_body);
|
let disabled = source_disables_thinking(original_request_body, provider_request_body);
|
||||||
set_deepseek_thinking_type(
|
set_deepseek_thinking_type(
|
||||||
provider_request_body,
|
provider_request_body,
|
||||||
@@ -170,33 +151,6 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
|||||||
{
|
{
|
||||||
object.remove("reasoning_effort");
|
object.remove("reasoning_effort");
|
||||||
}
|
}
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let Some(messages) = object.get_mut("messages").and_then(Value::as_array_mut) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
for message in messages {
|
|
||||||
let Some(message_object) = message.as_object_mut() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let is_assistant = message_object
|
|
||||||
.get("role")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"));
|
|
||||||
if !is_assistant {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if message_object
|
|
||||||
.get("reasoning_content")
|
|
||||||
.is_some_and(|value| !value.is_null())
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
message_object.insert(
|
|
||||||
"reasoning_content".to_string(),
|
|
||||||
Value::String(String::new()),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -287,8 +241,29 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn detects_deepseek_provider_by_type_or_host() {
|
fn xai_reasoning_policy_comes_from_provider_type() {
|
||||||
assert!(is_deepseek_provider(
|
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||||
|
assert_eq!(
|
||||||
|
openai_responses_reasoning_replay_policy(
|
||||||
|
"xai",
|
||||||
|
"https://custom.example/v1",
|
||||||
|
"grok-4.6"
|
||||||
|
),
|
||||||
|
OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
openai_responses_reasoning_replay_policy(
|
||||||
|
"openai",
|
||||||
|
"https://custom.example/v1",
|
||||||
|
"grok-4.6"
|
||||||
|
),
|
||||||
|
OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_deepseek_provider_only_by_official_host() {
|
||||||
|
assert!(!is_deepseek_provider(
|
||||||
"deepseek",
|
"deepseek",
|
||||||
"https://relay.example.com"
|
"https://relay.example.com"
|
||||||
));
|
));
|
||||||
@@ -298,6 +273,16 @@ mod tests {
|
|||||||
));
|
));
|
||||||
assert!(is_deepseek_provider("custom", "api.deepseek.com/v1"));
|
assert!(is_deepseek_provider("custom", "api.deepseek.com/v1"));
|
||||||
assert!(is_deepseek_provider("custom", "api.deepseek.com:443/v1"));
|
assert!(is_deepseek_provider("custom", "api.deepseek.com:443/v1"));
|
||||||
|
assert!(!is_deepseek_provider("custom", "https://deepseek.com"));
|
||||||
|
assert!(!is_deepseek_provider("custom", "deepseek.com/v1"));
|
||||||
|
assert!(is_deepseek_provider(
|
||||||
|
"custom",
|
||||||
|
" HTTPS://API.DEEPSEEK.COM:443/beta "
|
||||||
|
));
|
||||||
|
assert!(!is_deepseek_provider(
|
||||||
|
"deepseek",
|
||||||
|
"https://other.deepseek.com/v1"
|
||||||
|
));
|
||||||
assert!(!is_deepseek_provider(
|
assert!(!is_deepseek_provider(
|
||||||
"custom",
|
"custom",
|
||||||
"https://example.com/deepseek"
|
"https://example.com/deepseek"
|
||||||
@@ -319,6 +304,15 @@ mod tests {
|
|||||||
"https://evil.example/[email protected]"
|
"https://evil.example/[email protected]"
|
||||||
));
|
));
|
||||||
assert!(!is_deepseek_provider("custom", "ftp://api.deepseek.com/v1"));
|
assert!(!is_deepseek_provider("custom", "ftp://api.deepseek.com/v1"));
|
||||||
|
assert!(!is_deepseek_provider("deepseek", ""));
|
||||||
|
assert_eq!(
|
||||||
|
openai_responses_reasoning_replay_policy(
|
||||||
|
"deepseek",
|
||||||
|
"https://deepseek.com/v1",
|
||||||
|
"deepseek-chat",
|
||||||
|
),
|
||||||
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
openai_responses_reasoning_replay_policy(
|
openai_responses_reasoning_replay_policy(
|
||||||
"custom",
|
"custom",
|
||||||
@@ -341,7 +335,7 @@ mod tests {
|
|||||||
"https://api.b.ai/v1",
|
"https://api.b.ai/v1",
|
||||||
"deepseek-v4-flash",
|
"deepseek-v4-flash",
|
||||||
),
|
),
|
||||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
openai_responses_reasoning_replay_policy(
|
openai_responses_reasoning_replay_policy(
|
||||||
@@ -353,6 +347,33 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn official_deepseek_host_enables_compat_without_type_or_model_hints() {
|
||||||
|
for base_url in [
|
||||||
|
"https://api.deepseek.com/v1",
|
||||||
|
"https://api.deepseek.com/beta",
|
||||||
|
] {
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "mapped-model",
|
||||||
|
"messages": [{"role": "assistant", "content": "answer"}]
|
||||||
|
});
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
"custom",
|
||||||
|
base_url,
|
||||||
|
"openai:chat",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(body["thinking"]["type"], "enabled");
|
||||||
|
assert_eq!(
|
||||||
|
openai_responses_reasoning_replay_policy("custom", base_url, "mapped-model"),
|
||||||
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn custom_deepseek_host_preserves_production_shaped_opaque_reasoning_replay() {
|
fn custom_deepseek_host_preserves_production_shaped_opaque_reasoning_replay() {
|
||||||
let reasoning_items = (0..66)
|
let reasoning_items = (0..66)
|
||||||
@@ -444,7 +465,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn openai_chat_deepseek_adds_thinking_and_empty_reasoning_content() {
|
fn openai_chat_deepseek_enables_thinking_without_fabricating_reasoning() {
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
"model": "deepseek-chat",
|
"model": "deepseek-chat",
|
||||||
"messages": [
|
"messages": [
|
||||||
@@ -467,11 +488,11 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(body["thinking"]["type"], "enabled");
|
assert_eq!(body["thinking"]["type"], "enabled");
|
||||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
assert!(body["messages"][1].get("reasoning_content").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn custom_relay_deepseek_model_adds_chat_thinking_compat() {
|
fn custom_relay_deepseek_model_preserves_chat_request() {
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
"model": "deepseek-v4-flash",
|
"model": "deepseek-v4-flash",
|
||||||
"messages": [
|
"messages": [
|
||||||
@@ -484,6 +505,7 @@ mod tests {
|
|||||||
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
|
let original = body.clone();
|
||||||
|
|
||||||
apply_deepseek_tool_call_thinking_compat(
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
&mut body,
|
&mut body,
|
||||||
@@ -493,8 +515,211 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
assert_eq!(body, original);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn third_party_hosts_ignore_deepseek_type_and_model_hints() {
|
||||||
|
for provider_type in [
|
||||||
|
"custom",
|
||||||
|
"deepseek",
|
||||||
|
"deepseek_openai",
|
||||||
|
"deepseek_anthropic",
|
||||||
|
"deepseek_compatible",
|
||||||
|
] {
|
||||||
|
for provider_model in [
|
||||||
|
"other-model",
|
||||||
|
"deepseek-chat",
|
||||||
|
"deepseek-reasoner",
|
||||||
|
"deepseek-v3",
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"vendor/deepseek-chat",
|
||||||
|
"vendor:deepseek-reasoner",
|
||||||
|
] {
|
||||||
|
let base_url = "https://relay.example.com/v1";
|
||||||
|
assert!(!is_deepseek_provider(provider_type, base_url));
|
||||||
|
assert_eq!(
|
||||||
|
openai_responses_reasoning_replay_policy(
|
||||||
|
provider_type,
|
||||||
|
base_url,
|
||||||
|
provider_model
|
||||||
|
),
|
||||||
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||||
|
);
|
||||||
|
|
||||||
|
for api_format in ["openai:chat", "claude:messages"] {
|
||||||
|
let original = json!({
|
||||||
|
"model": provider_model,
|
||||||
|
"messages": [{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "answer",
|
||||||
|
"reasoning_content": "original plan"
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
let mut body = original.clone();
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
provider_type,
|
||||||
|
base_url,
|
||||||
|
api_format,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
body, original,
|
||||||
|
"{provider_type} / {provider_model} / {api_format}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_chat_deepseek_preserves_history_without_tools() {
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-chat",
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "Compare 9.11 and 9.8"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "9.8 is greater",
|
||||||
|
"reasoning_content": "Compare the decimal places.\n9.80 > 9.11."
|
||||||
|
},
|
||||||
|
{"role": "user", "content": "Explain again"},
|
||||||
|
{"role": "assistant", "content": "Compare 9.80 with 9.11"}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
let messages = body["messages"].clone();
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
"deepseek",
|
||||||
|
"https://api.deepseek.com/v1",
|
||||||
|
"openai:chat",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(body["messages"], messages);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_chat_deepseek_preserves_reasoning_across_all_tool_turns() {
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-chat",
|
||||||
|
"tools": [{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_weather",
|
||||||
|
"parameters": {"type": "object", "properties": {}}
|
||||||
|
}
|
||||||
|
}],
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "What is the weather?"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": null,
|
||||||
|
"reasoning_content": "Check the weather before answering.\nKeep this full plan.",
|
||||||
|
"tool_calls": [{
|
||||||
|
"id": "call_1",
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": "get_weather", "arguments": "{}"}
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
{"role": "tool", "tool_call_id": "call_1", "content": "Cloudy"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "It is cloudy",
|
||||||
|
"reasoning_content": "The weather result is available; summarize it."
|
||||||
|
},
|
||||||
|
{"role": "user", "content": "Should I take an umbrella?"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "An umbrella may be useful",
|
||||||
|
"reasoning_content": "Use the previous weather result without another tool call."
|
||||||
|
},
|
||||||
|
{"role": "user", "content": "Why?"}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
let messages = body["messages"].clone();
|
||||||
|
let tools = body["tools"].clone();
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
"deepseek",
|
||||||
|
"https://api.deepseek.com/v1",
|
||||||
|
"openai:chat",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(body["messages"], messages);
|
||||||
|
assert_eq!(body["tools"], tools);
|
||||||
|
assert_eq!(body["thinking"]["type"], "enabled");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_chat_deepseek_does_not_fabricate_missing_tool_reasoning() {
|
||||||
|
for tools in [
|
||||||
|
json!([]),
|
||||||
|
json!([{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "lookup",
|
||||||
|
"parameters": {"type": "object", "properties": {}}
|
||||||
|
}
|
||||||
|
}]),
|
||||||
|
] {
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-chat",
|
||||||
|
"tools": tools,
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "hi"},
|
||||||
|
{"role": "assistant", "content": "missing"},
|
||||||
|
{"role": "assistant", "content": "null", "reasoning_content": null},
|
||||||
|
{"role": "assistant", "content": "empty", "reasoning_content": ""},
|
||||||
|
{"role": "assistant", "content": "answer", "reasoning_content": "original plan"}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
let messages = body["messages"].clone();
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
"deepseek",
|
||||||
|
"https://api.deepseek.com/v1",
|
||||||
|
"openai:chat",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(body["messages"], messages);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_chat_deepseek_preserves_reasoning_prefix_without_tools() {
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-chat",
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "What is 1 + 1?"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"prefix": true,
|
||||||
|
"content": "",
|
||||||
|
"reasoning_content": "Start by adding one to one."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
let messages = body["messages"].clone();
|
||||||
|
|
||||||
|
apply_deepseek_tool_call_thinking_compat(
|
||||||
|
&mut body,
|
||||||
|
"deepseek",
|
||||||
|
"https://api.deepseek.com/beta",
|
||||||
|
"openai:chat",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(body["messages"], messages);
|
||||||
assert_eq!(body["thinking"]["type"], "enabled");
|
assert_eq!(body["thinking"]["type"], "enabled");
|
||||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSour
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
@@ -232,7 +232,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttempt
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -378,7 +378,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ use std::sync::Arc;
|
|||||||
use aether_contracts::ResolvedTransportProfile;
|
use aether_contracts::ResolvedTransportProfile;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::ai_serving::planner::antigravity::{
|
||||||
|
build_antigravity_v1internal_provider_request, AntigravityV1InternalRequestError,
|
||||||
|
AntigravityV1InternalRequestInput, ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME,
|
||||||
|
};
|
||||||
use crate::ai_serving::planner::candidate_preparation::{
|
use crate::ai_serving::planner::candidate_preparation::{
|
||||||
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
||||||
OauthPreparationContext,
|
OauthPreparationContext,
|
||||||
@@ -26,6 +30,7 @@ use crate::ai_serving::planner::standard::{
|
|||||||
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
||||||
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
||||||
};
|
};
|
||||||
|
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||||
use crate::ai_serving::transport::kiro::{
|
use crate::ai_serving::transport::kiro::{
|
||||||
build_kiro_provider_headers, build_kiro_provider_request_body,
|
build_kiro_provider_headers, build_kiro_provider_request_body,
|
||||||
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
||||||
@@ -587,7 +592,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
crate::ai_serving::hydrate_openai_response_history(
|
crate::ai_serving::hydrate_openai_response_history(
|
||||||
state.runtime_state(),
|
state,
|
||||||
body_json,
|
body_json,
|
||||||
spec_metadata.api_format,
|
spec_metadata.api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
@@ -837,6 +842,29 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
|||||||
.await);
|
.await);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if normalized_provider_api_format == "gemini:generate_content"
|
||||||
|
&& is_antigravity_provider_transport(transport)
|
||||||
|
{
|
||||||
|
return Ok(build_antigravity_cross_format_payload_parts(
|
||||||
|
state,
|
||||||
|
parts,
|
||||||
|
trace_id,
|
||||||
|
body_json,
|
||||||
|
input,
|
||||||
|
attempt,
|
||||||
|
transport,
|
||||||
|
spec_metadata.api_format,
|
||||||
|
provider_api_format,
|
||||||
|
prepared_candidate.mapped_model,
|
||||||
|
prepared_candidate.auth_header,
|
||||||
|
prepared_candidate.auth_value,
|
||||||
|
provider_request_body,
|
||||||
|
upstream_is_stream,
|
||||||
|
redaction.redacted,
|
||||||
|
)
|
||||||
|
.await);
|
||||||
|
}
|
||||||
|
|
||||||
if normalized_provider_api_format == "gemini:generate_content"
|
if normalized_provider_api_format == "gemini:generate_content"
|
||||||
&& is_gemini_cli_provider_transport(transport)
|
&& is_gemini_cli_provider_transport(transport)
|
||||||
{
|
{
|
||||||
@@ -963,6 +991,145 @@ fn apply_transport_request_body_semantics(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn build_antigravity_cross_format_payload_parts(
|
||||||
|
state: &AppState,
|
||||||
|
parts: &http::request::Parts,
|
||||||
|
trace_id: &str,
|
||||||
|
original_body_json: &serde_json::Value,
|
||||||
|
input: &LocalStandardDecisionInput,
|
||||||
|
attempt: &LocalStandardCandidateAttempt,
|
||||||
|
transport: &Arc<GatewayProviderTransportSnapshot>,
|
||||||
|
client_api_format: &str,
|
||||||
|
provider_api_format: &str,
|
||||||
|
mapped_model: String,
|
||||||
|
auth_header: String,
|
||||||
|
auth_value: String,
|
||||||
|
gemini_request_body: Value,
|
||||||
|
upstream_is_stream: bool,
|
||||||
|
request_redacted: bool,
|
||||||
|
) -> Option<LocalStandardCandidatePayloadParts> {
|
||||||
|
let candidate = &attempt.eligible.candidate;
|
||||||
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
let resolved =
|
||||||
|
match build_antigravity_v1internal_provider_request(AntigravityV1InternalRequestInput {
|
||||||
|
state,
|
||||||
|
parts,
|
||||||
|
transport,
|
||||||
|
trace_id,
|
||||||
|
mapped_model: &mapped_model,
|
||||||
|
provider_api_format,
|
||||||
|
auth_header: &auth_header,
|
||||||
|
auth_value: &auth_value,
|
||||||
|
request_headers: effective_headers,
|
||||||
|
original_request_body: original_body_json,
|
||||||
|
gemini_request_body: &gemini_request_body,
|
||||||
|
upstream_is_stream,
|
||||||
|
same_format: false,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(resolved) => resolved,
|
||||||
|
Err(AntigravityV1InternalRequestError::TransportUnsupported) => {
|
||||||
|
mark_skipped_local_standard_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_unsupported",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::EnvelopeUnsupported) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_extra_data(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"provider_request_body_build_failed",
|
||||||
|
request_body_build_failure_extra_data(
|
||||||
|
original_body_json,
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::UpstreamUrlUnavailable) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"upstream_url_missing",
|
||||||
|
CandidateFailureDiagnostic::upstream_url_missing(
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
"standard_family_antigravity_url",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::HeaderRulesApplyFailed) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_header_rules_apply_failed",
|
||||||
|
CandidateFailureDiagnostic::header_rules_apply_failed(
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
"standard_family_antigravity_headers",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut provider_request_headers = resolved.headers.headers;
|
||||||
|
apply_codex_openai_special_headers(
|
||||||
|
&mut provider_request_headers,
|
||||||
|
&resolved.body,
|
||||||
|
effective_headers,
|
||||||
|
resolved.transport.provider.provider_type.as_str(),
|
||||||
|
provider_api_format,
|
||||||
|
Some(trace_id),
|
||||||
|
resolved.transport.key.decrypted_auth_config.as_deref(),
|
||||||
|
);
|
||||||
|
request_identity_response_encoding_when_redacted(
|
||||||
|
&mut provider_request_headers,
|
||||||
|
request_redacted,
|
||||||
|
);
|
||||||
|
|
||||||
|
Some(LocalStandardCandidatePayloadParts {
|
||||||
|
auth_header: resolved.headers.auth_header,
|
||||||
|
auth_value: resolved.headers.auth_value,
|
||||||
|
mapped_model,
|
||||||
|
provider_api_format: provider_api_format.to_string(),
|
||||||
|
provider_request_body: resolved.body,
|
||||||
|
provider_request_headers,
|
||||||
|
upstream_url: resolved.upstream_url,
|
||||||
|
upstream_is_stream,
|
||||||
|
envelope_name: Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME),
|
||||||
|
transport: resolved.transport,
|
||||||
|
transport_profile: None,
|
||||||
|
request_redacted,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn build_gemini_cli_cross_format_payload_parts(
|
async fn build_gemini_cli_cross_format_payload_parts(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use crate::ai_serving::transport::apply_standard_provider_request_body_rules_wit
|
|||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
apply_codex_openai_responses_chat_body_edits,
|
apply_codex_openai_responses_chat_body_edits,
|
||||||
apply_openai_responses_compact_special_body_edits,
|
apply_openai_responses_compact_special_body_edits,
|
||||||
build_cross_format_openai_chat_request_body_with_model_directives as surface_build_cross_format_openai_chat_request_body,
|
build_cross_format_openai_chat_request_body_with_provider_context as surface_build_cross_format_openai_chat_request_body,
|
||||||
build_local_openai_chat_request_body_with_model_directives as surface_build_local_openai_chat_request_body,
|
build_local_openai_chat_request_body_with_model_directives as surface_build_local_openai_chat_request_body,
|
||||||
GatewayProviderTransportSnapshot,
|
GatewayProviderTransportSnapshot,
|
||||||
};
|
};
|
||||||
@@ -73,9 +73,11 @@ pub(crate) fn build_cross_format_openai_chat_request_body(
|
|||||||
let provider_request_body = surface_build_cross_format_openai_chat_request_body(
|
let provider_request_body = surface_build_cross_format_openai_chat_request_body(
|
||||||
body_json,
|
body_json,
|
||||||
mapped_model,
|
mapped_model,
|
||||||
|
provider_type,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
enable_model_directives,
|
enable_model_directives,
|
||||||
|
user_api_key_id,
|
||||||
)?;
|
)?;
|
||||||
let mut provider_request_body =
|
let mut provider_request_body =
|
||||||
apply_standard_provider_request_body_rules_with_request_headers(
|
apply_standard_provider_request_body_rules_with_request_headers(
|
||||||
@@ -125,3 +127,39 @@ pub(crate) fn build_cross_format_openai_chat_upstream_url(
|
|||||||
parts.uri.query(),
|
parts.uri.query(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod antigravity_schema_tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn antigravity_chat_route_preserves_tool_schema_and_alternate_responses_shape() {
|
||||||
|
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||||
|
let body = json!({"model": "client", "messages": [{"role": "user", "content": "hi"}],
|
||||||
|
"tools": [{"type": "function", "function": {"name": "probe", "parameters": schema}}]});
|
||||||
|
let responses_body = json!({"model": "client", "input": "hi",
|
||||||
|
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||||
|
for input in [body, responses_body] {
|
||||||
|
for provider in ["antigravity", "gemini"] {
|
||||||
|
let output = build_cross_format_openai_chat_request_body(
|
||||||
|
&input,
|
||||||
|
"claude-test",
|
||||||
|
provider,
|
||||||
|
"gemini:generate_content",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
&http::HeaderMap::new(),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||||
|
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||||
|
assert!(output.get("stream").is_none());
|
||||||
|
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
|||||||
use crate::ai_serving::transport::apply_standard_provider_request_body_rules_with_request_headers;
|
use crate::ai_serving::transport::apply_standard_provider_request_body_rules_with_request_headers;
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
apply_openai_responses_compact_special_body_edits,
|
apply_openai_responses_compact_special_body_edits,
|
||||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope as surface_build_cross_format_openai_responses_request_body,
|
build_cross_format_openai_responses_request_body_with_provider_context as surface_build_cross_format_openai_responses_request_body,
|
||||||
build_local_openai_responses_request_body_with_model_directives as surface_build_local_openai_responses_request_body,
|
build_local_openai_responses_request_body_with_model_directives as surface_build_local_openai_responses_request_body,
|
||||||
GatewayProviderTransportSnapshot,
|
GatewayProviderTransportSnapshot,
|
||||||
};
|
};
|
||||||
@@ -218,6 +218,7 @@ pub(crate) fn build_cross_format_openai_responses_request_body_with_codex_model_
|
|||||||
body_json,
|
body_json,
|
||||||
mapped_model,
|
mapped_model,
|
||||||
client_api_format,
|
client_api_format,
|
||||||
|
provider_type,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
enable_model_directives,
|
enable_model_directives,
|
||||||
@@ -291,3 +292,38 @@ pub(crate) fn build_cross_format_openai_responses_upstream_url(
|
|||||||
parts.uri.query(),
|
parts.uri.query(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod antigravity_schema_tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn antigravity_responses_route_preserves_tool_schema_without_changing_public_gemini() {
|
||||||
|
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||||
|
let input = json!({"model": "client", "input": "hi",
|
||||||
|
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||||
|
for provider in ["antigravity", "gemini"] {
|
||||||
|
let output =
|
||||||
|
build_cross_format_openai_responses_request_body_with_codex_model_capabilities(
|
||||||
|
&input,
|
||||||
|
"claude-test",
|
||||||
|
"openai:responses",
|
||||||
|
"gemini:generate_content",
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
provider,
|
||||||
|
None,
|
||||||
|
&http::HeaderMap::new(),
|
||||||
|
Some("antigravity-schema-test"),
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||||
|
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||||
|
assert!(output.get("stream").is_none());
|
||||||
|
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+16
-6
@@ -140,7 +140,7 @@ fn finalize_openai_chat_provider_request_body(
|
|||||||
mapped_model,
|
mapped_model,
|
||||||
source_model,
|
source_model,
|
||||||
);
|
);
|
||||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
let finalization_failure = crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||||
provider_request_body,
|
provider_request_body,
|
||||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||||
source_api_format: "openai:chat",
|
source_api_format: "openai:chat",
|
||||||
@@ -170,7 +170,17 @@ fn finalize_openai_chat_provider_request_body(
|
|||||||
provider_api_format,
|
provider_api_format,
|
||||||
"openai_chat_request_finalization",
|
"openai_chat_request_finalization",
|
||||||
)
|
)
|
||||||
})
|
});
|
||||||
|
if finalization_failure.is_none() {
|
||||||
|
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||||
|
// Claude Code body mimicry must be applied here for Chat -> claude_code requests.
|
||||||
|
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||||
|
provider_request_body,
|
||||||
|
transport,
|
||||||
|
provider_api_format,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
finalization_failure
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
@@ -2183,7 +2193,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -2741,7 +2751,7 @@ mod tests {
|
|||||||
.provider_request_headers
|
.provider_request_headers
|
||||||
.get("x-client-version")
|
.get("x-client-version")
|
||||||
.map(String::as_str),
|
.map(String::as_str),
|
||||||
Some("1.2.3")
|
Some("4.3.0")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
payload
|
payload
|
||||||
@@ -2761,9 +2771,9 @@ mod tests {
|
|||||||
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
payload.provider_request_body["userAgent"],
|
payload.provider_request_body["userAgent"],
|
||||||
"antigravity/cli/1.0.16 (aidev_client; os_type=linux; arch=arm64; auth_method=consumer)"
|
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||||
);
|
);
|
||||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
assert!(payload.provider_request_body.get("requestType").is_none());
|
||||||
assert!(payload.provider_request_body.get("contents").is_none());
|
assert!(payload.provider_request_body.get("contents").is_none());
|
||||||
assert!(payload.provider_request_body["request"]
|
assert!(payload.provider_request_body["request"]
|
||||||
.get("contents")
|
.get("contents")
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use aether_routing_core::RoutingExecutionPolicy;
|
use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
@@ -124,7 +124,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttem
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
@@ -207,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
|
|||||||
async fn next_raw_attempt_with_target_select(
|
async fn next_raw_attempt_with_target_select(
|
||||||
&mut self,
|
&mut self,
|
||||||
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
||||||
let select_window = openai_chat_stream_target_select_window();
|
let select_window = openai_chat_stream_target_select_window_for_mode(
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.scheduling_mode),
|
||||||
|
);
|
||||||
if select_window <= 1 {
|
if select_window <= 1 {
|
||||||
return self.next_raw_attempt_linear().await;
|
return self.next_raw_attempt_linear().await;
|
||||||
}
|
}
|
||||||
@@ -365,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize {
|
|||||||
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn openai_chat_stream_target_select_window_for_mode(
|
||||||
|
scheduling_mode: Option<RoutingSchedulingMode>,
|
||||||
|
) -> usize {
|
||||||
|
if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
openai_chat_stream_target_select_window()
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct TargetSelectCandidateIdentity<'a> {
|
struct TargetSelectCandidateIdentity<'a> {
|
||||||
provider_id: &'a str,
|
provider_id: &'a str,
|
||||||
@@ -574,4 +588,14 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(select_target_index(19, &choices), 1);
|
assert_eq!(select_target_index(19, &choices), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixed_order_disables_stream_target_selection() {
|
||||||
|
assert_eq!(
|
||||||
|
openai_chat_stream_target_select_window_for_mode(Some(
|
||||||
|
RoutingSchedulingMode::FixedOrder,
|
||||||
|
)),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSo
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
|
|||||||
+12
-27
@@ -1,7 +1,6 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use aether_contracts::RequestBody;
|
use aether_contracts::RequestBody;
|
||||||
use tracing::debug;
|
|
||||||
|
|
||||||
use super::super::{
|
use super::super::{
|
||||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||||
@@ -10,7 +9,6 @@ use super::super::{
|
|||||||
AiStreamAttempt,
|
AiStreamAttempt,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let (url, url_source) = if let Some(upstream_url) =
|
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||||
take_non_empty_string(&mut payload.upstream_url)
|
upstream_url
|
||||||
{
|
|
||||||
(upstream_url, "upstream_url")
|
|
||||||
} else {
|
} else {
|
||||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
(
|
build_standard_plan_fallback_openai_responses_url(
|
||||||
build_standard_plan_fallback_openai_responses_url(
|
&upstream_base_url,
|
||||||
&upstream_base_url,
|
parts.uri.query(),
|
||||||
parts.uri.query(),
|
compact,
|
||||||
compact,
|
|
||||||
),
|
|
||||||
"upstream_base_url",
|
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_decision_upstream_base_url = payload
|
tracing::debug!(
|
||||||
.upstream_base_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_decision_upstream_url = payload
|
|
||||||
.upstream_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
|
||||||
debug!(
|
|
||||||
event_name = "local_openai_responses_stream_plan_built",
|
event_name = "local_openai_responses_stream_plan_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
request_id = %plan.request_id,
|
request_id = %plan.request_id,
|
||||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
provider_id = %plan.provider_id,
|
provider_id = %plan.provider_id,
|
||||||
endpoint_id = %plan.endpoint_id,
|
endpoint_id = %plan.endpoint_id,
|
||||||
key_id = %plan.key_id,
|
key_id = %plan.key_id,
|
||||||
|
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
downstream_query = ?log_downstream_query,
|
downstream_query = ?log_downstream_query,
|
||||||
url_source,
|
|
||||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
|
||||||
decision_upstream_url = ?log_decision_upstream_url,
|
|
||||||
plan_url = %log_plan_url,
|
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
upstream_is_stream = effective_upstream_is_stream,
|
upstream_is_stream = effective_upstream_is_stream,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use aether_contracts::RequestBody;
|
use aether_contracts::RequestBody;
|
||||||
use tracing::debug;
|
|
||||||
|
|
||||||
use super::super::{
|
use super::super::{
|
||||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||||
@@ -10,7 +9,6 @@ use super::super::{
|
|||||||
AiSyncAttempt,
|
AiSyncAttempt,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let (url, url_source) = if let Some(upstream_url) =
|
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||||
take_non_empty_string(&mut payload.upstream_url)
|
upstream_url
|
||||||
{
|
|
||||||
(upstream_url, "upstream_url")
|
|
||||||
} else {
|
} else {
|
||||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
(
|
build_standard_plan_fallback_openai_responses_url(
|
||||||
build_standard_plan_fallback_openai_responses_url(
|
&upstream_base_url,
|
||||||
&upstream_base_url,
|
parts.uri.query(),
|
||||||
parts.uri.query(),
|
compact,
|
||||||
compact,
|
|
||||||
),
|
|
||||||
"upstream_base_url",
|
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_decision_upstream_base_url = payload
|
tracing::debug!(
|
||||||
.upstream_base_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_decision_upstream_url = payload
|
|
||||||
.upstream_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
|
||||||
debug!(
|
|
||||||
event_name = "local_openai_responses_sync_plan_built",
|
event_name = "local_openai_responses_sync_plan_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
request_id = %plan.request_id,
|
request_id = %plan.request_id,
|
||||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
provider_id = %plan.provider_id,
|
provider_id = %plan.provider_id,
|
||||||
endpoint_id = %plan.endpoint_id,
|
endpoint_id = %plan.endpoint_id,
|
||||||
key_id = %plan.key_id,
|
key_id = %plan.key_id,
|
||||||
|
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
downstream_query = ?log_downstream_query,
|
downstream_query = ?log_downstream_query,
|
||||||
url_source,
|
|
||||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
|
||||||
decision_upstream_url = ?log_decision_upstream_url,
|
|
||||||
plan_url = %log_plan_url,
|
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
upstream_is_stream = payload.upstream_is_stream,
|
upstream_is_stream = payload.upstream_is_stream,
|
||||||
|
|||||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
|||||||
|
|
||||||
use crate::ai_serving::build_request_trace_proxy_value;
|
use crate::ai_serving::build_request_trace_proxy_value;
|
||||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::planner::report_context::{
|
use crate::ai_serving::planner::report_context::{
|
||||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
|||||||
&resolved.transport,
|
&resolved.transport,
|
||||||
);
|
);
|
||||||
|
|
||||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
|
||||||
let log_request_query = parts
|
let log_request_query = parts
|
||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_decision_payload_built",
|
event_name = "local_openai_responses_decision_payload_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
|||||||
client_api_format = spec_metadata.api_format,
|
client_api_format = spec_metadata.api_format,
|
||||||
provider_api_format = %resolved.provider_api_format,
|
provider_api_format = %resolved.provider_api_format,
|
||||||
request_path = %parts.uri.path(),
|
request_path = %parts.uri.path(),
|
||||||
|
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||||
request_query = ?log_request_query,
|
request_query = ?log_request_query,
|
||||||
upstream_base_url = %log_base_url,
|
|
||||||
upstream_url = %log_upstream_url,
|
|
||||||
upstream_is_stream = resolved.upstream_is_stream,
|
upstream_is_stream = resolved.upstream_is_stream,
|
||||||
has_envelope = resolved.envelope_name.is_some(),
|
has_envelope = resolved.envelope_name.is_some(),
|
||||||
"gateway built local openai responses decision payload"
|
"gateway built local openai responses decision payload"
|
||||||
|
|||||||
+14
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
|||||||
};
|
};
|
||||||
use crate::ai_serving::planner::redaction::{
|
use crate::ai_serving::planner::redaction::{
|
||||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||||
sanitize_upstream_url_for_log,
|
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||||
use crate::ai_serving::planner::standard::{
|
use crate::ai_serving::planner::standard::{
|
||||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
crate::ai_serving::hydrate_openai_response_history(
|
crate::ai_serving::hydrate_openai_response_history(
|
||||||
state.runtime_state(),
|
state,
|
||||||
body_json,
|
body_json,
|
||||||
spec_metadata.api_format,
|
spec_metadata.api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
@@ -636,6 +635,13 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
{
|
{
|
||||||
log_responses_to_chat_tool_conversion(trace_id, body_json, &base_provider_request_body);
|
log_responses_to_chat_tool_conversion(trace_id, body_json, &base_provider_request_body);
|
||||||
}
|
}
|
||||||
|
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||||
|
// Claude Code body mimicry must be applied here for Responses -> claude_code requests.
|
||||||
|
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||||
|
&mut base_provider_request_body,
|
||||||
|
&transport,
|
||||||
|
provider_api_format,
|
||||||
|
);
|
||||||
let provider_request_body = base_provider_request_body;
|
let provider_request_body = base_provider_request_body;
|
||||||
|
|
||||||
if let Some(kiro_auth) = kiro_auth.as_ref() {
|
if let Some(kiro_auth) = kiro_auth.as_ref() {
|
||||||
@@ -867,17 +873,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
|
|
||||||
let (execution_strategy, conversion_mode) =
|
let (execution_strategy, conversion_mode) =
|
||||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
|
||||||
let log_custom_path = transport
|
|
||||||
.endpoint
|
|
||||||
.custom_path
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_request_query = parts
|
let log_request_query = parts
|
||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_upstream_url_resolved",
|
event_name = "local_openai_responses_upstream_url_resolved",
|
||||||
@@ -893,12 +892,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
provider_api_format = %provider_api_format,
|
provider_api_format = %provider_api_format,
|
||||||
execution_strategy = execution_strategy.as_str(),
|
execution_strategy = execution_strategy.as_str(),
|
||||||
conversion_mode = conversion_mode.as_str(),
|
conversion_mode = conversion_mode.as_str(),
|
||||||
base_url = %log_base_url,
|
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
custom_path = ?log_custom_path,
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||||
request_path = %parts.uri.path(),
|
request_path = %parts.uri.path(),
|
||||||
request_query = ?log_request_query,
|
request_query = ?log_request_query,
|
||||||
mapped_model = %mapped_model,
|
mapped_model = %mapped_model,
|
||||||
upstream_url = %log_upstream_url,
|
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
"gateway resolved local openai responses upstream url"
|
"gateway resolved local openai responses upstream url"
|
||||||
);
|
);
|
||||||
@@ -2011,8 +2012,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
|||||||
};
|
};
|
||||||
let (execution_strategy, conversion_mode) =
|
let (execution_strategy, conversion_mode) =
|
||||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
@@ -2027,7 +2026,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
|||||||
provider_api_format = %provider_api_format,
|
provider_api_format = %provider_api_format,
|
||||||
execution_strategy = execution_strategy.as_str(),
|
execution_strategy = execution_strategy.as_str(),
|
||||||
conversion_mode = conversion_mode.as_str(),
|
conversion_mode = conversion_mode.as_str(),
|
||||||
upstream_url = %log_upstream_url,
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
"gateway resolved local openai responses kiro upstream url"
|
"gateway resolved local openai responses kiro upstream url"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -488,6 +488,7 @@ impl ResponsesWebSocketBodyNormalization {
|
|||||||
digest.update([match self.reasoning_replay_policy {
|
digest.update([match self.reasoning_replay_policy {
|
||||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds => 0,
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds => 0,
|
||||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque => 1,
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque => 1,
|
||||||
|
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted => 2,
|
||||||
}]);
|
}]);
|
||||||
update_normalization_optional_json_digest(&mut digest, self.model_directive_patch.as_ref());
|
update_normalization_optional_json_digest(&mut digest, self.model_directive_patch.as_ref());
|
||||||
digest.finalize().into()
|
digest.finalize().into()
|
||||||
@@ -729,117 +730,6 @@ fn update_normalization_codex_capabilities_digest(
|
|||||||
update_normalization_string_vec_digest(digest, &capabilities.supported_service_tiers);
|
update_normalization_string_vec_digest(digest, &capabilities.supported_service_tiers);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod continuation_fingerprint_tests {
|
|
||||||
use http::HeaderValue;
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
use super::ResponsesWebSocketBodyNormalization;
|
|
||||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
|
||||||
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
|
||||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
|
||||||
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
|
||||||
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
|
||||||
assert_eq!(
|
|
||||||
first.continuation_fingerprint(),
|
|
||||||
second.continuation_fingerprint()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn normalization_fingerprint_changes_with_effective_contract() {
|
|
||||||
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
|
||||||
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
|
||||||
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
|
||||||
);
|
|
||||||
assert_ne!(
|
|
||||||
base.continuation_fingerprint(),
|
|
||||||
changed_policy.continuation_fingerprint()
|
|
||||||
);
|
|
||||||
|
|
||||||
let changed_patch = base
|
|
||||||
.clone()
|
|
||||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
|
||||||
assert_ne!(
|
|
||||||
base.continuation_fingerprint(),
|
|
||||||
changed_patch.continuation_fingerprint()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
|
||||||
let body_rules = json!([{
|
|
||||||
"action": "set",
|
|
||||||
"path": "store",
|
|
||||||
"value": false,
|
|
||||||
"condition": {
|
|
||||||
"source": "request_headers",
|
|
||||||
"path": "x-contract",
|
|
||||||
"op": "eq",
|
|
||||||
"value": "enabled"
|
|
||||||
}
|
|
||||||
}]);
|
|
||||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
|
||||||
.with_body_rules_for_tests(body_rules);
|
|
||||||
first
|
|
||||||
.request_headers
|
|
||||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
|
||||||
first
|
|
||||||
.request_headers
|
|
||||||
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
|
||||||
first
|
|
||||||
.request_headers
|
|
||||||
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
|
||||||
let mut second = first.clone();
|
|
||||||
second
|
|
||||||
.request_headers
|
|
||||||
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
|
||||||
second
|
|
||||||
.request_headers
|
|
||||||
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
first.continuation_fingerprint(),
|
|
||||||
second.continuation_fingerprint(),
|
|
||||||
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
|
||||||
let body_rules = json!([{
|
|
||||||
"action": "set",
|
|
||||||
"path": "store",
|
|
||||||
"value": false,
|
|
||||||
"condition": {
|
|
||||||
"source": "request_headers",
|
|
||||||
"path": "X-Contract",
|
|
||||||
"op": "eq",
|
|
||||||
"value": "enabled"
|
|
||||||
}
|
|
||||||
}]);
|
|
||||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
|
||||||
.with_body_rules_for_tests(body_rules.clone());
|
|
||||||
first
|
|
||||||
.request_headers
|
|
||||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
|
||||||
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
|
||||||
.with_body_rules_for_tests(body_rules);
|
|
||||||
second
|
|
||||||
.request_headers
|
|
||||||
.insert("x-contract", HeaderValue::from_static("disabled"));
|
|
||||||
|
|
||||||
assert_ne!(
|
|
||||||
first.continuation_fingerprint(),
|
|
||||||
second.continuation_fingerprint(),
|
|
||||||
"a header that controls an effective body-rule condition remains part of the contract"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Builds one upstream decision for a Responses WebSocket turn. The session
|
/// Builds one upstream decision for a Responses WebSocket turn. The session
|
||||||
/// reuses this decision for same-model turns and invokes the planner again when
|
/// reuses this decision for same-model turns and invokes the planner again when
|
||||||
/// a later `response.create` changes the public model.
|
/// a later `response.create` changes the public model.
|
||||||
@@ -1058,3 +948,114 @@ async fn release_responses_websocket_planning_lease(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod continuation_fingerprint_tests {
|
||||||
|
use http::HeaderValue;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::ResponsesWebSocketBodyNormalization;
|
||||||
|
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
||||||
|
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||||
|
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
||||||
|
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||||
|
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
||||||
|
assert_eq!(
|
||||||
|
first.continuation_fingerprint(),
|
||||||
|
second.continuation_fingerprint()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalization_fingerprint_changes_with_effective_contract() {
|
||||||
|
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
||||||
|
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
||||||
|
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
base.continuation_fingerprint(),
|
||||||
|
changed_policy.continuation_fingerprint()
|
||||||
|
);
|
||||||
|
|
||||||
|
let changed_patch = base
|
||||||
|
.clone()
|
||||||
|
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
||||||
|
assert_ne!(
|
||||||
|
base.continuation_fingerprint(),
|
||||||
|
changed_patch.continuation_fingerprint()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
||||||
|
let body_rules = json!([{
|
||||||
|
"action": "set",
|
||||||
|
"path": "store",
|
||||||
|
"value": false,
|
||||||
|
"condition": {
|
||||||
|
"source": "request_headers",
|
||||||
|
"path": "x-contract",
|
||||||
|
"op": "eq",
|
||||||
|
"value": "enabled"
|
||||||
|
}
|
||||||
|
}]);
|
||||||
|
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||||
|
.with_body_rules_for_tests(body_rules);
|
||||||
|
first
|
||||||
|
.request_headers
|
||||||
|
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||||
|
first
|
||||||
|
.request_headers
|
||||||
|
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
||||||
|
first
|
||||||
|
.request_headers
|
||||||
|
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
||||||
|
let mut second = first.clone();
|
||||||
|
second
|
||||||
|
.request_headers
|
||||||
|
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
||||||
|
second
|
||||||
|
.request_headers
|
||||||
|
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
first.continuation_fingerprint(),
|
||||||
|
second.continuation_fingerprint(),
|
||||||
|
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
||||||
|
let body_rules = json!([{
|
||||||
|
"action": "set",
|
||||||
|
"path": "store",
|
||||||
|
"value": false,
|
||||||
|
"condition": {
|
||||||
|
"source": "request_headers",
|
||||||
|
"path": "X-Contract",
|
||||||
|
"op": "eq",
|
||||||
|
"value": "enabled"
|
||||||
|
}
|
||||||
|
}]);
|
||||||
|
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||||
|
.with_body_rules_for_tests(body_rules.clone());
|
||||||
|
first
|
||||||
|
.request_headers
|
||||||
|
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||||
|
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||||
|
.with_body_rules_for_tests(body_rules);
|
||||||
|
second
|
||||||
|
.request_headers
|
||||||
|
.insert("x-contract", HeaderValue::from_static("disabled"));
|
||||||
|
|
||||||
|
assert_ne!(
|
||||||
|
first.continuation_fingerprint(),
|
||||||
|
second.continuation_fingerprint(),
|
||||||
|
"a header that controls an effective body-rule condition remains part of the contract"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -166,7 +166,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAtte
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
@@ -216,7 +216,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStream
|
|||||||
self.input
|
self.input
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|policy| policy.execution_policy)
|
.map(|policy| policy.execution_policy.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate};
|
use aether_scheduler_core::{ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::time::Instant;
|
|
||||||
|
|
||||||
use super::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
use super::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
||||||
use crate::clock::current_unix_secs;
|
|
||||||
use crate::constants::{
|
use crate::constants::{
|
||||||
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
|
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
|
||||||
};
|
};
|
||||||
@@ -97,11 +95,13 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
),
|
),
|
||||||
GatewayError,
|
GatewayError,
|
||||||
> {
|
> {
|
||||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
self.app(),
|
||||||
let wait_deadline = Instant::now() + wait_timeout;
|
auth_snapshot,
|
||||||
let mut attempt_now_unix_secs = now_unix_secs;
|
now_unix_secs,
|
||||||
loop {
|
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||||
|
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||||
|
|attempt_now_unix_secs| async move {
|
||||||
let result = crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons_for_request_operation(
|
let result = crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons_for_request_operation(
|
||||||
self.app().data.as_ref(),
|
self.app().data.as_ref(),
|
||||||
self.app(),
|
self.app(),
|
||||||
@@ -118,21 +118,13 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if !crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
let auth_limit_blocked = crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
||||||
&result.0, &result.1,
|
&result.0, &result.1,
|
||||||
) {
|
);
|
||||||
return Ok(result);
|
Ok((result, auth_limit_blocked))
|
||||||
}
|
},
|
||||||
|
)
|
||||||
let now = Instant::now();
|
.await
|
||||||
if now >= wait_deadline {
|
|
||||||
return Ok(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
let remaining = wait_deadline.duration_since(now);
|
|
||||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
|
||||||
attempt_now_unix_secs = current_unix_secs();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
@@ -178,13 +170,14 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
ordering_config: SchedulerOrderingConfig,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
self.app(),
|
||||||
let wait_deadline = Instant::now() + wait_timeout;
|
auth_snapshot,
|
||||||
let mut attempt_now_unix_secs = now_unix_secs;
|
now_unix_secs,
|
||||||
|
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||||
loop {
|
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||||
let (result, auth_limit_blocked) = crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
|attempt_now_unix_secs| {
|
||||||
|
crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
||||||
self.app().data.as_ref(),
|
self.app().data.as_ref(),
|
||||||
self.app(),
|
self.app(),
|
||||||
candidate_api_format,
|
candidate_api_format,
|
||||||
@@ -195,20 +188,8 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
attempt_now_unix_secs,
|
attempt_now_unix_secs,
|
||||||
ordering_config,
|
ordering_config,
|
||||||
)
|
)
|
||||||
.await?;
|
},
|
||||||
|
)
|
||||||
if !auth_limit_blocked {
|
.await
|
||||||
return Ok(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
let now = Instant::now();
|
|
||||||
if now >= wait_deadline {
|
|
||||||
return Ok(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
let remaining = wait_deadline.duration_since(now);
|
|
||||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
|
||||||
attempt_now_unix_secs = current_unix_secs();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,13 +12,16 @@ pub(crate) use aether_ai_formats::api::{
|
|||||||
apply_codex_openai_responses_websocket_continuation_body_edits_with_source_model_and_capabilities,
|
apply_codex_openai_responses_websocket_continuation_body_edits_with_source_model_and_capabilities,
|
||||||
apply_codex_openai_special_headers, apply_model_directive_mapping_patch,
|
apply_codex_openai_special_headers, apply_model_directive_mapping_patch,
|
||||||
apply_model_directive_overrides_from_model, apply_model_directive_overrides_from_request,
|
apply_model_directive_overrides_from_model, apply_model_directive_overrides_from_request,
|
||||||
apply_openai_responses_compact_special_body_edits, build_chatgpt_web_image_request_body,
|
apply_openai_responses_compact_special_body_edits, apply_xai_upstream_payload_edits,
|
||||||
|
apply_xai_upstream_payload_edits_with_client, build_chatgpt_web_image_request_body,
|
||||||
build_codex_model_catalog_metadata, build_codex_openai_image_api_provider_request_body,
|
build_codex_model_catalog_metadata, build_codex_openai_image_api_provider_request_body,
|
||||||
build_core_error_body_for_client_format, build_cross_format_openai_chat_request_body,
|
build_core_error_body_for_client_format, build_cross_format_openai_chat_request_body,
|
||||||
build_cross_format_openai_chat_request_body_with_model_directives,
|
build_cross_format_openai_chat_request_body_with_model_directives,
|
||||||
|
build_cross_format_openai_chat_request_body_with_provider_context,
|
||||||
build_cross_format_openai_responses_request_body,
|
build_cross_format_openai_responses_request_body,
|
||||||
build_cross_format_openai_responses_request_body_with_model_directives,
|
build_cross_format_openai_responses_request_body_with_model_directives,
|
||||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope,
|
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope,
|
||||||
|
build_cross_format_openai_responses_request_body_with_provider_context,
|
||||||
build_gemini_image_request_body_from_openai_image_request,
|
build_gemini_image_request_body_from_openai_image_request,
|
||||||
build_gemini_image_response_from_openai_image_response,
|
build_gemini_image_response_from_openai_image_response,
|
||||||
build_gemini_image_response_from_openai_responses_image_response, build_generated_tool_call_id,
|
build_gemini_image_response_from_openai_responses_image_response, build_generated_tool_call_id,
|
||||||
@@ -160,7 +163,7 @@ pub(crate) use aether_ai_formats::api::{
|
|||||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND,
|
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND,
|
||||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
||||||
OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
||||||
OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||||
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
||||||
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||||
@@ -177,10 +180,11 @@ pub(crate) use aether_ai_formats::{
|
|||||||
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
||||||
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
||||||
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
||||||
|
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
|
||||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||||
strip_incompatible_openai_responses_reasoning_items,
|
strip_incompatible_openai_responses_reasoning_items,
|
||||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||||
CODEX_CLIENT_VERSION, OPENAI_RESPONSES_OPERATION_COMPACT,
|
OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn plan_kind_matches_api_operation(
|
pub(crate) fn plan_kind_matches_api_operation(
|
||||||
|
|||||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
|||||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||||
};
|
};
|
||||||
use aether_runtime_state::RuntimeState;
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::GatewayError;
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
|
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||||
|
|
||||||
pub(crate) async fn hydrate_openai_response_history(
|
pub(crate) async fn hydrate_openai_response_history(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
request: &Value,
|
request: &Value,
|
||||||
client_api_format: &str,
|
client_api_format: &str,
|
||||||
provider_api_format: &str,
|
provider_api_format: &str,
|
||||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||||
|
let runtime_state = state.runtime_state();
|
||||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||||
warn!(
|
warn!(
|
||||||
event_name = "openai_response_history_read_failed",
|
event_name = "openai_response_history_read_failed",
|
||||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
let Some(payload) = payload else {
|
let Some(payload) = payload else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||||
|
state,
|
||||||
|
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||||
|
&payload,
|
||||||
|
) else {
|
||||||
|
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||||
|
warn!(
|
||||||
|
event_name = "openai_response_history_decryption_failed",
|
||||||
|
log_type = "ops",
|
||||||
|
backend = runtime_state.backend_kind().as_str(),
|
||||||
|
"gateway rejected undecryptable shared OpenAI response history"
|
||||||
|
);
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"OpenAI response history decryption failed".to_string(),
|
||||||
|
));
|
||||||
|
};
|
||||||
if let Err(error) =
|
if let Err(error) =
|
||||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||||
{
|
{
|
||||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||||
warn!(
|
warn!(
|
||||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn persist_response_history_record(
|
pub(crate) async fn persist_response_history_record(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
record: ResponseHistoryRecord,
|
record: ResponseHistoryRecord,
|
||||||
) {
|
) {
|
||||||
|
let runtime_state = state.runtime_state();
|
||||||
|
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||||
|
state,
|
||||||
|
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||||
|
&record.payload,
|
||||||
|
) else {
|
||||||
|
warn!(
|
||||||
|
event_name = "openai_response_history_encryption_unavailable",
|
||||||
|
log_type = "ops",
|
||||||
|
backend = runtime_state.backend_kind().as_str(),
|
||||||
|
"gateway refused to persist unencrypted OpenAI response history"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
if let Err(error) = runtime_state
|
if let Err(error) = runtime_state
|
||||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
warn!(
|
warn!(
|
||||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn persist_converted_response_history(
|
pub(crate) async fn persist_converted_response_history(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
report_context: &Value,
|
report_context: &Value,
|
||||||
response: Option<&Value>,
|
response: Option<&Value>,
|
||||||
) {
|
) {
|
||||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||||
persist_response_history_record(runtime_state, record).await;
|
persist_response_history_record(state, record).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||||
|
use serde_json::json;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||||
|
};
|
||||||
|
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||||
|
|
||||||
|
fn response_history_test_state() -> AppState {
|
||||||
|
AppState::new()
|
||||||
|
.expect("test state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
)
|
||||||
|
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||||
|
MemoryRuntimeStateConfig::default(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||||
|
let expires_at_unix_secs = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
.saturating_add(3600);
|
||||||
|
json!({
|
||||||
|
"version": 1,
|
||||||
|
"response_id": response_id,
|
||||||
|
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||||
|
"expires_at_unix_secs": expires_at_unix_secs,
|
||||||
|
"transcript": [{"type": "message", "content": marker}],
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||||
|
let state = response_history_test_state();
|
||||||
|
let response_id = "resp_gateway_encrypted_history_v1";
|
||||||
|
let scope = "response-history-encrypted-scope";
|
||||||
|
let marker = "private-response-history-marker";
|
||||||
|
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||||
|
let payload = response_history_payload(response_id, scope, marker);
|
||||||
|
|
||||||
|
persist_response_history_record(
|
||||||
|
&state,
|
||||||
|
ResponseHistoryRecord {
|
||||||
|
storage_key: storage_key.clone(),
|
||||||
|
payload,
|
||||||
|
ttl: Duration::from_secs(6 * 60 * 60),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let stored = state
|
||||||
|
.runtime_kv_get(&storage_key)
|
||||||
|
.await
|
||||||
|
.expect("history lookup should succeed")
|
||||||
|
.expect("history should be persisted");
|
||||||
|
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||||
|
&stored
|
||||||
|
));
|
||||||
|
assert!(!stored.contains(marker));
|
||||||
|
|
||||||
|
hydrate_openai_response_history(
|
||||||
|
&state,
|
||||||
|
&json!({"previous_response_id": response_id}),
|
||||||
|
"openai:responses",
|
||||||
|
"openai:chat",
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("encrypted history should hydrate");
|
||||||
|
assert!(crate::ai_serving::response_history_is_loaded(
|
||||||
|
response_id,
|
||||||
|
Some(scope)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||||
|
let state = response_history_test_state();
|
||||||
|
let response_id = "resp_gateway_legacy_history_v1";
|
||||||
|
let scope = "response-history-legacy-scope";
|
||||||
|
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||||
|
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||||
|
state
|
||||||
|
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||||
|
.await
|
||||||
|
.expect("legacy history should store");
|
||||||
|
|
||||||
|
let result = hydrate_openai_response_history(
|
||||||
|
&state,
|
||||||
|
&json!({"previous_response_id": response_id}),
|
||||||
|
"openai:responses",
|
||||||
|
"openai:chat",
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||||
|
response_id,
|
||||||
|
Some(scope)
|
||||||
|
));
|
||||||
|
assert!(state
|
||||||
|
.runtime_kv_get(&storage_key)
|
||||||
|
.await
|
||||||
|
.expect("history lookup should succeed")
|
||||||
|
.is_none());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,11 +58,16 @@ pub(crate) mod windsurf {
|
|||||||
pub(crate) use aether_provider_transport::windsurf::*;
|
pub(crate) use aether_provider_transport::windsurf::*;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) mod xai {
|
||||||
|
pub(crate) use aether_provider_transport::xai::*;
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) use aether_provider_transport::{
|
pub(crate) use aether_provider_transport::{
|
||||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||||
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
||||||
apply_local_header_rules_with_request_headers, apply_standard_provider_request_body_rules,
|
apply_local_header_rules_with_request_headers, apply_provider_outbound_request_policies,
|
||||||
|
apply_standard_provider_request_body_rules,
|
||||||
apply_standard_provider_request_body_rules_with_request_headers,
|
apply_standard_provider_request_body_rules_with_request_headers,
|
||||||
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
||||||
body_rules_handle_path, body_rules_have_enabled_rules,
|
body_rules_handle_path, body_rules_have_enabled_rules,
|
||||||
@@ -73,7 +78,7 @@ pub(crate) use aether_provider_transport::{
|
|||||||
build_local_openai_chat_upstream_url, build_local_openai_responses_upstream_url,
|
build_local_openai_chat_upstream_url, build_local_openai_responses_upstream_url,
|
||||||
build_openai_image_headers, build_openai_image_upstream_url, build_passthrough_headers,
|
build_openai_image_headers, build_openai_image_upstream_url, build_passthrough_headers,
|
||||||
build_request_trace_proxy_value, build_same_format_provider_headers,
|
build_request_trace_proxy_value, build_same_format_provider_headers,
|
||||||
build_same_format_provider_request_body,
|
build_same_format_provider_request_body, build_same_format_provider_request_body_for_operation,
|
||||||
build_same_format_provider_request_body_with_compatibility_report,
|
build_same_format_provider_request_body_with_compatibility_report,
|
||||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy,
|
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy,
|
||||||
build_same_format_provider_upstream_url, build_standard_plan_fallback_headers,
|
build_same_format_provider_upstream_url, build_standard_plan_fallback_headers,
|
||||||
@@ -112,7 +117,11 @@ pub(crate) use aether_provider_transport::{
|
|||||||
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
||||||
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
||||||
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
||||||
ProviderOpenAiImageHeadersInput, ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
ProviderOpenAiImageHeadersInput, ProviderOutboundRequestContext,
|
||||||
|
ProviderOutboundRequestIdentityScope, ProviderOutboundRequestMutationScope,
|
||||||
|
ProviderOutboundRequestPolicy, ProviderOutboundRequestPolicyOutcome,
|
||||||
|
ProviderOutboundRequestPolicyReason, ProviderOutboundRequestPolicyResult,
|
||||||
|
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||||
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
||||||
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
||||||
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
||||||
@@ -121,5 +130,5 @@ pub(crate) use aether_provider_transport::{
|
|||||||
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
||||||
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
||||||
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
||||||
GROK_RATE_LIMITS_PATH, WINDSURF_ENVELOPE_NAME,
|
GROK_RATE_LIMITS_PATH, PROVIDER_OUTBOUND_CONTEXT_MAX_VALUE_BYTES, WINDSURF_ENVELOPE_NAME,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -53,6 +53,8 @@ const AI_ANY_ROUTE_PATTERNS: &[&str] = &[
|
|||||||
"/v1beta/operations/{*operation_path}",
|
"/v1beta/operations/{*operation_path}",
|
||||||
"/v1/videos",
|
"/v1/videos",
|
||||||
"/v1/videos/{*video_path}",
|
"/v1/videos/{*video_path}",
|
||||||
|
"/openai/v1/videos",
|
||||||
|
"/openai/v1/videos/{*video_path}",
|
||||||
"/upload/v1beta/files",
|
"/upload/v1beta/files",
|
||||||
"/v1beta/files",
|
"/v1beta/files",
|
||||||
"/v1beta/files/{*file_path}",
|
"/v1beta/files/{*file_path}",
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
|||||||
"internal_gateway": {
|
"internal_gateway": {
|
||||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||||
"status": "rust_native_control_plane",
|
"status": state.internal_gateway_auth_status(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"features": {
|
"features": {
|
||||||
|
|||||||
@@ -1,5 +1,14 @@
|
|||||||
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
|
use axum::body::Body;
|
||||||
|
use axum::extract::{ConnectInfo, Request, State};
|
||||||
|
use axum::http::{self, HeaderValue, StatusCode};
|
||||||
|
use axum::middleware::{self, Next};
|
||||||
|
use axum::response::{IntoResponse, Response};
|
||||||
use axum::routing::{get, post};
|
use axum::routing::{get, post};
|
||||||
use axum::Router;
|
use axum::{Json, Router};
|
||||||
|
use serde_json::json;
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::async_task::{
|
use crate::async_task::{
|
||||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
|||||||
use crate::router::metrics;
|
use crate::router::metrics;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
#[derive(Clone, Copy)]
|
||||||
router
|
struct OperationalPermission {
|
||||||
|
required_permissions: &'static [&'static str],
|
||||||
|
write: bool,
|
||||||
|
requires_full_admin_role: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn mount_operational_routes(
|
||||||
|
router: Router<AppState>,
|
||||||
|
state: AppState,
|
||||||
|
) -> Router<AppState> {
|
||||||
|
let operational = Router::<AppState>::new()
|
||||||
.route("/_gateway/metrics", get(metrics))
|
.route("/_gateway/metrics", get(metrics))
|
||||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||||
.route(
|
.route(
|
||||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
|||||||
"/_gateway/audit/request-usage/{request_id}",
|
"/_gateway/audit/request-usage/{request_id}",
|
||||||
get(get_request_usage_audit),
|
get(get_request_usage_audit),
|
||||||
)
|
)
|
||||||
|
.route_layer(middleware::from_fn_with_state(
|
||||||
|
state,
|
||||||
|
authorize_operational_request,
|
||||||
|
));
|
||||||
|
router.merge(operational)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn authorize_operational_request(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
request: Request,
|
||||||
|
next: Next,
|
||||||
|
) -> Response<Body> {
|
||||||
|
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"operational route permission is not configured",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let Some(remote_addr) = request
|
||||||
|
.extensions()
|
||||||
|
.get::<ConnectInfo<SocketAddr>>()
|
||||||
|
.map(|value| value.0)
|
||||||
|
else {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let headers = request.headers().clone();
|
||||||
|
let uri = request.uri().clone();
|
||||||
|
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||||
|
return operational_auth_required_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||||
|
Ok(Some(principal)) => {
|
||||||
|
if permission.requires_full_admin_role
|
||||||
|
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||||
|
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||||
|
&state, &headers, client_ip,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(authenticated) => authenticated,
|
||||||
|
Err(
|
||||||
|
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||||
|
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||||
|
) => return operational_auth_required_response(),
|
||||||
|
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if permission.requires_full_admin_role
|
||||||
|
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
let missing_permission =
|
||||||
|
permission
|
||||||
|
.required_permissions
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.find(|required| {
|
||||||
|
!management_token_has_operational_permission(
|
||||||
|
&authenticated.permissions,
|
||||||
|
required,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
if let Some(required_permission) = missing_permission {
|
||||||
|
return operational_permission_denied_response(required_permission);
|
||||||
|
}
|
||||||
|
|
||||||
|
let client_ip = client_ip.to_string();
|
||||||
|
if let Err(err) = state
|
||||||
|
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
token_id = %authenticated.token.id,
|
||||||
|
error = ?err,
|
||||||
|
"gateway failed to record operational management token usage"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut response = next.run(request).await;
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::CACHE_CONTROL,
|
||||||
|
HeaderValue::from_static("no-store"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||||
|
if path == "/_gateway/metrics" {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:monitoring:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||||
|
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: if write {
|
||||||
|
&["admin:video_tasks:write"]
|
||||||
|
} else {
|
||||||
|
&["admin:video_tasks:read"]
|
||||||
|
},
|
||||||
|
write,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:api_keys:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &[
|
||||||
|
"admin:monitoring:admin",
|
||||||
|
"admin:usage:read",
|
||||||
|
"admin:api_keys:read",
|
||||||
|
],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||||
|
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||||
|
{
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:monitoring:admin"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:usage:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn management_token_has_operational_permission(
|
||||||
|
permissions: &[String],
|
||||||
|
required_permission: &str,
|
||||||
|
) -> bool {
|
||||||
|
let scope = required_permission
|
||||||
|
.rsplit_once(':')
|
||||||
|
.map(|(scope, _)| scope)
|
||||||
|
.unwrap_or(required_permission);
|
||||||
|
let admin_permission = format!("{scope}:admin");
|
||||||
|
permissions
|
||||||
|
.iter()
|
||||||
|
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_auth_required_response() -> Response<Body> {
|
||||||
|
let mut response = operational_error_response(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
"admin authentication required",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::WWW_AUTHENTICATE,
|
||||||
|
HeaderValue::from_static("Bearer"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||||
|
operational_error_response(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"operational permission denied",
|
||||||
|
Some(required_permission),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_error_response(
|
||||||
|
status: StatusCode,
|
||||||
|
detail: &'static str,
|
||||||
|
required_permission: Option<&'static str>,
|
||||||
|
) -> Response<Body> {
|
||||||
|
let mut response = (
|
||||||
|
status,
|
||||||
|
Json(json!({
|
||||||
|
"detail": detail,
|
||||||
|
"required_permission": required_permission,
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::CACHE_CONTROL,
|
||||||
|
HeaderValue::from_static("no-store"),
|
||||||
|
);
|
||||||
|
response
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
|||||||
use crate::control::GatewayControlDecision;
|
use crate::control::GatewayControlDecision;
|
||||||
use crate::control::GatewayLocalAuthRejection;
|
use crate::control::GatewayLocalAuthRejection;
|
||||||
use crate::headers::should_skip_response_header;
|
use crate::headers::should_skip_response_header;
|
||||||
|
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||||
use crate::{insert_header_if_missing, GatewayError};
|
use crate::{insert_header_if_missing, GatewayError};
|
||||||
|
|
||||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||||
|
// Provider responses are API data, even when an untrusted provider labels
|
||||||
|
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||||
|
// from becoming same-origin active content, and prevent referrer leakage
|
||||||
|
// if a user follows a link rendered from such a response.
|
||||||
|
headers.insert(
|
||||||
|
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
headers.insert(
|
||||||
|
HeaderName::from_static("content-security-policy"),
|
||||||
|
HeaderValue::from_static(
|
||||||
|
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
headers.insert(
|
||||||
|
HeaderName::from_static("referrer-policy"),
|
||||||
|
HeaderValue::from_static("no-referrer"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_client_response(
|
pub(crate) fn build_client_response(
|
||||||
upstream_response: reqwest::Response,
|
upstream_response: reqwest::Response,
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
) -> Result<Response<Body>, GatewayError> {
|
) -> Result<Response<Body>, GatewayError> {
|
||||||
let status = upstream_response.status();
|
let status = upstream_response.status();
|
||||||
let upstream_headers = upstream_response
|
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||||
.headers()
|
|
||||||
.iter()
|
|
||||||
.map(|(name, value)| {
|
|
||||||
(
|
|
||||||
name.as_str().to_string(),
|
|
||||||
value.to_str().unwrap_or_default().to_string(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let upstream_stream = upstream_response.bytes_stream();
|
let upstream_stream = upstream_response.bytes_stream();
|
||||||
build_client_response_from_parts(
|
build_client_response_from_parts(
|
||||||
status.as_u16(),
|
status.as_u16(),
|
||||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||||
|
let connection_declared = aether_http::connection_declared_header_names(
|
||||||
|
headers
|
||||||
|
.get_all(http::header::CONNECTION)
|
||||||
|
.iter()
|
||||||
|
.filter_map(|value| value.to_str().ok()),
|
||||||
|
);
|
||||||
|
headers
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, value)| {
|
||||||
|
let normalized = name.as_str().to_ascii_lowercase();
|
||||||
|
if should_skip_client_response_header(&normalized)
|
||||||
|
|| connection_declared.contains(&normalized)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.ok()
|
||||||
|
.map(|value| (normalized, value.to_string()))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_skip_client_response_header(name: &str) -> bool {
|
||||||
|
should_skip_response_header(name)
|
||||||
|
// A provider Location is relative to the provider, not to the gateway.
|
||||||
|
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||||
|
// can disclose their gateway Authorization header to another origin.
|
||||||
|
// Keep Location available inside execution reports, but never expose
|
||||||
|
// it on the client-facing response boundary.
|
||||||
|
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_client_response_from_parts(
|
pub(crate) fn build_client_response_from_parts(
|
||||||
status_code: u16,
|
status_code: u16,
|
||||||
upstream_headers: &BTreeMap<String, String>,
|
upstream_headers: &BTreeMap<String, String>,
|
||||||
@@ -111,8 +158,17 @@ where
|
|||||||
.body(body)
|
.body(body)
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
|
||||||
|
let connection_declared = aether_http::connection_declared_header_names(
|
||||||
|
upstream_headers
|
||||||
|
.iter()
|
||||||
|
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||||
|
.map(|(_, value)| value.as_str()),
|
||||||
|
);
|
||||||
|
|
||||||
for (name, value) in upstream_headers {
|
for (name, value) in upstream_headers {
|
||||||
if should_skip_response_header(name.as_str()) {
|
if should_skip_client_response_header(name.as_str())
|
||||||
|
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||||
@@ -123,6 +179,7 @@ where
|
|||||||
}
|
}
|
||||||
mutate_headers(response.headers_mut())?;
|
mutate_headers(response.headers_mut())?;
|
||||||
apply_streaming_response_headers(response.headers_mut());
|
apply_streaming_response_headers(response.headers_mut());
|
||||||
|
apply_gateway_browser_security_headers(response.headers_mut());
|
||||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||||
if let Some(decision) = control_decision {
|
if let Some(decision) = control_decision {
|
||||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn build_local_plan_usage_limited_response(
|
||||||
|
trace_id: &str,
|
||||||
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
|
rejection: &PlanUsagePolicyRejection,
|
||||||
|
) -> Result<Response<Body>, GatewayError> {
|
||||||
|
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||||
|
let fallback_payload = json!({
|
||||||
|
"error": {
|
||||||
|
"type": "plan_usage_limit_exceeded",
|
||||||
|
"message": message,
|
||||||
|
"details": {
|
||||||
|
"metric": rejection.metric,
|
||||||
|
"window": rejection.window,
|
||||||
|
"limit": rejection.limit,
|
||||||
|
"retry_after": rejection.retry_after,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let payload = build_local_error_payload(
|
||||||
|
control_decision,
|
||||||
|
None,
|
||||||
|
message,
|
||||||
|
LocalCoreSyncErrorKind::RateLimit,
|
||||||
|
fallback_payload,
|
||||||
|
);
|
||||||
|
let body =
|
||||||
|
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
let headers = BTreeMap::from([
|
||||||
|
("content-type".to_string(), "application/json".to_string()),
|
||||||
|
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||||
|
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||||
|
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||||
|
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||||
|
(
|
||||||
|
"X-RateLimit-Metric".to_string(),
|
||||||
|
rejection.metric.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"X-RateLimit-Window".to_string(),
|
||||||
|
rejection.window.to_string(),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
build_client_response_from_parts(
|
||||||
|
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||||
|
&headers,
|
||||||
|
Body::from(body),
|
||||||
|
trace_id,
|
||||||
|
control_decision,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_local_http_error_response(
|
pub(crate) fn build_local_http_error_response(
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
build_client_response, build_client_response_from_parts,
|
||||||
|
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||||
build_local_user_rpm_limited_response,
|
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||||
};
|
};
|
||||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||||
|
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||||
use axum::body::{to_bytes, Body};
|
use axum::body::{to_bytes, Body};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -490,6 +600,163 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||||
|
let response = build_client_response_from_parts_with_mutator(
|
||||||
|
200,
|
||||||
|
&BTreeMap::from([
|
||||||
|
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||||
|
(
|
||||||
|
"x-aether-gateway".to_string(),
|
||||||
|
"attacker-gateway".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-aether-control-action".to_string(),
|
||||||
|
"attacker-action".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-aether-future-control".to_string(),
|
||||||
|
"attacker-future".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-accel-redirect".to_string(),
|
||||||
|
"/internal/private-file".to_string(),
|
||||||
|
),
|
||||||
|
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||||
|
(
|
||||||
|
"x-reproxy-url".to_string(),
|
||||||
|
"http://127.0.0.1:9000/private".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"access-control-allow-origin".to_string(),
|
||||||
|
"https://attacker.example".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"access-control-allow-credentials".to_string(),
|
||||||
|
"true".to_string(),
|
||||||
|
),
|
||||||
|
("content-length".to_string(), "999999".to_string()),
|
||||||
|
(
|
||||||
|
"content-security-policy".to_string(),
|
||||||
|
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"content-security-policy-report-only".to_string(),
|
||||||
|
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"reporting-endpoints".to_string(),
|
||||||
|
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||||
|
),
|
||||||
|
("report-to".to_string(), "attacker".to_string()),
|
||||||
|
(
|
||||||
|
"nel".to_string(),
|
||||||
|
"{\"report_to\":\"attacker\"}".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"refresh".to_string(),
|
||||||
|
"0; url=https://attacker.example".to_string(),
|
||||||
|
),
|
||||||
|
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||||
|
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||||
|
(
|
||||||
|
"location".to_string(),
|
||||||
|
"https://provider.example/direct".to_string(),
|
||||||
|
),
|
||||||
|
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||||
|
]),
|
||||||
|
Body::empty(),
|
||||||
|
"trace-upstream-header-filter",
|
||||||
|
None,
|
||||||
|
|headers| {
|
||||||
|
headers.insert(
|
||||||
|
http::HeaderName::from_static("x-aether-control-action"),
|
||||||
|
http::HeaderValue::from_static("gateway-action"),
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("response should build");
|
||||||
|
|
||||||
|
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||||
|
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||||
|
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||||
|
assert!(response.headers().get("x-sendfile").is_none());
|
||||||
|
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("access-control-allow-origin")
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("access-control-allow-credentials")
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get(http::header::CONTENT_LENGTH)
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("content-security-policy-report-only")
|
||||||
|
.is_none());
|
||||||
|
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||||
|
assert!(response.headers().get("report-to").is_none());
|
||||||
|
assert!(response.headers().get("nel").is_none());
|
||||||
|
assert!(response.headers().get("refresh").is_none());
|
||||||
|
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()["content-security-policy"],
|
||||||
|
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||||
|
"nosniff"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()["x-aether-control-action"],
|
||||||
|
"gateway-action"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||||
|
.await
|
||||||
|
.expect("listener");
|
||||||
|
let addr = listener.local_addr().expect("address");
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||||
|
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||||
|
let mut request = [0_u8; 1024];
|
||||||
|
let _ = stream.read(&mut request).await.expect("request read");
|
||||||
|
stream
|
||||||
|
.write_all(
|
||||||
|
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("response write");
|
||||||
|
});
|
||||||
|
let upstream = reqwest::Client::builder()
|
||||||
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
.build()
|
||||||
|
.expect("client")
|
||||||
|
.get(format!("http://{addr}/"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("upstream response");
|
||||||
|
|
||||||
|
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||||
|
.expect("client response");
|
||||||
|
server.await.expect("server");
|
||||||
|
|
||||||
|
assert!(response.headers().get("connection").is_none());
|
||||||
|
assert!(response.headers().get("x-first-hop").is_none());
|
||||||
|
assert!(response.headers().get("x-second-hop").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
fn claude_decision() -> GatewayControlDecision {
|
fn claude_decision() -> GatewayControlDecision {
|
||||||
GatewayControlDecision::synthetic(
|
GatewayControlDecision::synthetic(
|
||||||
"/v1/messages",
|
"/v1/messages",
|
||||||
@@ -581,4 +848,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||||
|
let response = build_local_plan_usage_limited_response(
|
||||||
|
"trace-plan-limit",
|
||||||
|
None,
|
||||||
|
&PlanUsagePolicyRejection {
|
||||||
|
metric: "request_count",
|
||||||
|
limit: 100.0,
|
||||||
|
retry_after: 42,
|
||||||
|
window: "calendar_week",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("response");
|
||||||
|
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||||
|
assert_eq!(response.headers()["retry-after"], "42");
|
||||||
|
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||||
|
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||||
|
let payload = response_json(response).await;
|
||||||
|
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use aether_contracts::ExecutionResult;
|
use aether_contracts::ExecutionResult;
|
||||||
@@ -21,7 +22,9 @@ use super::{
|
|||||||
};
|
};
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
pub(crate) use self::cancel::{
|
||||||
|
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub(crate) struct ListVideoTasksQuery {
|
pub(crate) struct ListVideoTasksQuery {
|
||||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn build_video_task_video_response(
|
pub(crate) async fn build_video_task_video_response(
|
||||||
state: &AppState,
|
_state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
source: VideoTaskVideoSource,
|
source: VideoTaskVideoSource,
|
||||||
) -> Result<axum::response::Response, GatewayError> {
|
) -> Result<axum::response::Response, GatewayError> {
|
||||||
match source {
|
match source {
|
||||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
VideoTaskVideoSource::Redirect { url } => {
|
||||||
|
resolve_public_video_target(&url).await?;
|
||||||
|
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||||
|
}
|
||||||
VideoTaskVideoSource::Proxy {
|
VideoTaskVideoSource::Proxy {
|
||||||
url,
|
url,
|
||||||
header_name,
|
header_name,
|
||||||
header_value,
|
header_value,
|
||||||
filename,
|
filename,
|
||||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn proxy_video_stream(
|
async fn proxy_video_stream(
|
||||||
state: &AppState,
|
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
url: &str,
|
url: &url::Url,
|
||||||
header_name: &str,
|
header_name: &str,
|
||||||
header_value: &str,
|
header_value: &str,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
) -> Result<axum::response::Response, GatewayError> {
|
) -> Result<axum::response::Response, GatewayError> {
|
||||||
let response = state
|
let target = resolve_public_video_target(url).await?;
|
||||||
.client
|
let client = build_pinned_video_client(&target)?;
|
||||||
.get(url)
|
let response = client
|
||||||
|
.get(url.clone())
|
||||||
.header(header_name, header_value)
|
.header(header_name, header_value)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||||
trace_id: task_id.to_string(),
|
trace_id: task_id.to_string(),
|
||||||
message: err.to_string(),
|
message: video_request_failure_message(&err).to_string(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if response.status().is_client_error() || response.status().is_server_error() {
|
if response.status().is_redirection() {
|
||||||
|
return Err(GatewayError::UpstreamUnavailable {
|
||||||
|
trace_id: task_id.to_string(),
|
||||||
|
message: format!(
|
||||||
|
"video upstream redirect was rejected with HTTP {}",
|
||||||
|
response.status()
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if !response.status().is_success() {
|
||||||
return Err(GatewayError::UpstreamUnavailable {
|
return Err(GatewayError::UpstreamUnavailable {
|
||||||
trace_id: task_id.to_string(),
|
trace_id: task_id.to_string(),
|
||||||
message: format!("video upstream returned HTTP {}", response.status()),
|
message: format!("video upstream returned HTTP {}", response.status()),
|
||||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let content_type = response
|
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||||
.headers()
|
// Reqwest may decode transfer/content encodings and the provider controls
|
||||||
.get(axum::http::header::CONTENT_TYPE)
|
// the declaration; forwarding a stale value would make the client-facing
|
||||||
.cloned()
|
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
// will select safe framing for the actual stream.
|
||||||
let content_length = response
|
let upstream_headers = response.headers().clone();
|
||||||
.headers()
|
|
||||||
.get(axum::http::header::CONTENT_LENGTH)
|
|
||||||
.cloned();
|
|
||||||
let cache_control = response
|
|
||||||
.headers()
|
|
||||||
.get(axum::http::header::CACHE_CONTROL)
|
|
||||||
.cloned();
|
|
||||||
let body = Body::from_stream(response.bytes_stream());
|
let body = Body::from_stream(response.bytes_stream());
|
||||||
|
|
||||||
let mut outbound = axum::http::Response::builder()
|
let mut outbound = axum::http::Response::builder()
|
||||||
.status(status)
|
.status(status)
|
||||||
.body(body)
|
.body(body)
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
outbound
|
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
|
||||||
outbound.headers_mut().insert(
|
|
||||||
axum::http::header::CONTENT_DISPOSITION,
|
|
||||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
|
||||||
);
|
|
||||||
if let Some(content_length) = content_length {
|
|
||||||
outbound
|
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
|
||||||
}
|
|
||||||
if let Some(cache_control) = cache_control {
|
|
||||||
outbound
|
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
|
||||||
} else {
|
|
||||||
outbound.headers_mut().insert(
|
|
||||||
axum::http::header::CACHE_CONTROL,
|
|
||||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(outbound)
|
Ok(outbound)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_safe_video_response_metadata(
|
||||||
|
outbound: &mut axum::http::HeaderMap,
|
||||||
|
upstream: &axum::http::HeaderMap,
|
||||||
|
filename: &str,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
let content_type = upstream
|
||||||
|
.get(axum::http::header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.and_then(safe_video_content_type)
|
||||||
|
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||||
|
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::CONTENT_DISPOSITION,
|
||||||
|
axum::http::HeaderValue::from_str(&format!(
|
||||||
|
"inline; filename=\"{}\"",
|
||||||
|
safe_video_filename(filename)
|
||||||
|
))
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||||
|
);
|
||||||
|
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::CACHE_CONTROL,
|
||||||
|
axum::http::HeaderValue::from_static("private, no-store"),
|
||||||
|
);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
axum::http::HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||||
|
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||||
|
let subtype = media_type.strip_prefix("video/")?;
|
||||||
|
if subtype.is_empty()
|
||||||
|
|| !subtype.bytes().all(|byte| {
|
||||||
|
byte.is_ascii_alphanumeric()
|
||||||
|
|| matches!(
|
||||||
|
byte,
|
||||||
|
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe_video_filename(filename: &str) -> String {
|
||||||
|
let filename = filename
|
||||||
|
.chars()
|
||||||
|
.take(255)
|
||||||
|
.map(|character| {
|
||||||
|
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||||
|
character
|
||||||
|
} else {
|
||||||
|
'_'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<String>();
|
||||||
|
if filename.is_empty() {
|
||||||
|
"video.mp4".to_string()
|
||||||
|
} else {
|
||||||
|
filename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ResolvedVideoTarget {
|
||||||
|
host: String,
|
||||||
|
addrs: Vec<SocketAddr>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
{
|
||||||
|
return Err(video_target_rejected(
|
||||||
|
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let port = url
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||||
|
let (host, addrs) = match url.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||||
|
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||||
|
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||||
|
let addrs = aether_http::lookup_host_with_limits(
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||||
|
(host.to_string(), addrs)
|
||||||
|
}
|
||||||
|
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||||
|
};
|
||||||
|
if addrs.is_empty()
|
||||||
|
|| addrs
|
||||||
|
.iter()
|
||||||
|
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||||
|
{
|
||||||
|
return Err(video_target_rejected(
|
||||||
|
"video URL resolves to a private or reserved address",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(ResolvedVideoTarget { host, addrs })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_pinned_video_client(
|
||||||
|
target: &ResolvedVideoTarget,
|
||||||
|
) -> Result<reqwest::Client, GatewayError> {
|
||||||
|
let mut builder = aether_http::apply_http_client_config(
|
||||||
|
reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.redirect(reqwest::redirect::Policy::none()),
|
||||||
|
&aether_http::HttpClientConfig {
|
||||||
|
connect_timeout_ms: Some(10_000),
|
||||||
|
request_timeout_ms: Some(300_000),
|
||||||
|
http2_adaptive_window: true,
|
||||||
|
..aether_http::HttpClientConfig::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if target.host.parse::<IpAddr>().is_err() {
|
||||||
|
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||||
|
}
|
||||||
|
builder
|
||||||
|
.build()
|
||||||
|
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_target_rejected(message: &str) -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: message.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||||
|
if error.is_timeout() {
|
||||||
|
"video upstream request timed out"
|
||||||
|
} else if error.is_connect() {
|
||||||
|
"video upstream connection failed"
|
||||||
|
} else if error.is_body() || error.is_decode() {
|
||||||
|
"video upstream response failed"
|
||||||
|
} else {
|
||||||
|
"video upstream request failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use axum::response::IntoResponse;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||||
|
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||||
|
VideoTaskVideoSource,
|
||||||
|
};
|
||||||
|
use crate::AppState;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_redirect_response_accepts_public_target() {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
let target = "https://8.8.8.8/video.mp4";
|
||||||
|
|
||||||
|
let response = build_video_task_video_response(
|
||||||
|
&state,
|
||||||
|
"task-public-redirect",
|
||||||
|
VideoTaskVideoSource::Redirect {
|
||||||
|
url: url::Url::parse(target).expect("public target should parse"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("public redirect should build");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response.status(),
|
||||||
|
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(axum::http::header::LOCATION)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some(target)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
|
||||||
|
for raw_url in [
|
||||||
|
"http://127.0.0.1/video.mp4",
|
||||||
|
"http://169.254.169.254/latest/meta-data",
|
||||||
|
"http://10.0.0.1/video.mp4",
|
||||||
|
"http://[::1]/video.mp4",
|
||||||
|
] {
|
||||||
|
let error = build_video_task_video_response(
|
||||||
|
&state,
|
||||||
|
"task-rejected-redirect",
|
||||||
|
VideoTaskVideoSource::Redirect {
|
||||||
|
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("private or reserved redirect target should be rejected");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.into_response().status(),
|
||||||
|
axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
"unexpected status for {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||||
|
for raw_url in [
|
||||||
|
"http://127.0.0.1/video.mp4",
|
||||||
|
"http://169.254.169.254/latest/meta-data",
|
||||||
|
"http://10.0.0.1/video.mp4",
|
||||||
|
"http://[::1]/video.mp4",
|
||||||
|
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||||
|
] {
|
||||||
|
let url = url::Url::parse(raw_url).unwrap();
|
||||||
|
assert!(
|
||||||
|
resolve_public_video_target(&url).await.is_err(),
|
||||||
|
"target should be rejected: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||||
|
for raw_url in [
|
||||||
|
"https://8.8.8.8/video.mp4",
|
||||||
|
"https://[2606:4700:4700::1111]/video.mp4",
|
||||||
|
] {
|
||||||
|
let url = url::Url::parse(raw_url).unwrap();
|
||||||
|
assert!(
|
||||||
|
resolve_public_video_target(&url).await.is_ok(),
|
||||||
|
"target should be accepted: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||||
|
assert!(safe_video_content_type("video/mp4").is_some());
|
||||||
|
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||||
|
assert!(safe_video_content_type("video/").is_none());
|
||||||
|
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||||
|
assert!(safe_video_content_type("text/html").is_none());
|
||||||
|
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||||
|
assert_eq!(
|
||||||
|
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||||
|
"video_123.mp4___filename__attack.html"
|
||||||
|
);
|
||||||
|
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||||
|
|
||||||
|
let mut upstream = axum::http::HeaderMap::new();
|
||||||
|
upstream.insert(
|
||||||
|
axum::http::header::CONTENT_TYPE,
|
||||||
|
axum::http::HeaderValue::from_static("text/html"),
|
||||||
|
);
|
||||||
|
upstream.insert(
|
||||||
|
axum::http::header::CONTENT_LENGTH,
|
||||||
|
axum::http::HeaderValue::from_static("999999"),
|
||||||
|
);
|
||||||
|
let mut outbound = upstream.clone();
|
||||||
|
apply_safe_video_response_metadata(
|
||||||
|
&mut outbound,
|
||||||
|
&upstream,
|
||||||
|
"video.mp4\"; filename=\"attack.html",
|
||||||
|
)
|
||||||
|
.expect("video metadata should build");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("application/octet-stream")
|
||||||
|
);
|
||||||
|
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("nosniff")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
|||||||
};
|
};
|
||||||
use axum::response::IntoResponse;
|
use axum::response::IntoResponse;
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::json;
|
||||||
|
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
use super::super::finalize_video_task_if_terminal;
|
use super::super::finalize_video_task_if_terminal;
|
||||||
use super::super::read_video_task_detail;
|
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||||
use super::current_unix_secs;
|
use super::current_unix_secs;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
|||||||
state: &AppState,
|
state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
cancel_video_task_record_inner(state, task_id, None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn cancel_video_task_record_for_user(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
|
let user_id = user_id.trim();
|
||||||
|
if user_id.is_empty() {
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
}
|
||||||
|
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_video_task_record_inner(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
expected_user_id: Option<&str>,
|
||||||
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
|
let task = match expected_user_id {
|
||||||
|
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||||
|
None => read_video_task_detail(state, task_id).await?,
|
||||||
|
};
|
||||||
|
let Some(task) = task else {
|
||||||
return Err(CancelVideoTaskError::NotFound);
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||||
|
let mut finalize_mutation = None;
|
||||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||||
state
|
|
||||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let body_json = json!({});
|
let body_json = json!({});
|
||||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
let follow_up = if let Some(user_id) = expected_user_id {
|
||||||
cancel_plan.plan_kind,
|
if state
|
||||||
&cancel_plan.request_path,
|
.hydrate_video_task_for_route_for_user(
|
||||||
Some(&body_json),
|
Some(cancel_plan.route_family),
|
||||||
None,
|
&cancel_plan.request_path,
|
||||||
&trace_id,
|
user_id,
|
||||||
);
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
}
|
||||||
|
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||||
|
cancel_plan.plan_kind,
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
Some(&body_json),
|
||||||
|
user_id,
|
||||||
|
task.api_key_id.as_deref(),
|
||||||
|
&trace_id,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
.hydrate_video_task_for_route(
|
||||||
|
Some(cancel_plan.route_family),
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
state.video_tasks.prepare_follow_up_sync_plan(
|
||||||
|
cancel_plan.plan_kind,
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
Some(&body_json),
|
||||||
|
None,
|
||||||
|
&trace_id,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
if let Some(follow_up) = follow_up {
|
if let Some(follow_up) = follow_up {
|
||||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||||
.await
|
.await
|
||||||
.map_err(CancelVideoTaskError::Response)?;
|
.map_err(CancelVideoTaskError::Response)?;
|
||||||
|
finalize_mutation = Some((
|
||||||
|
cancel_plan.request_path,
|
||||||
|
cancel_plan.report_kind.to_string(),
|
||||||
|
));
|
||||||
|
} else if expected_user_id.is_none() {
|
||||||
|
finalize_mutation = Some((
|
||||||
|
cancel_plan.request_path,
|
||||||
|
cancel_plan.report_kind.to_string(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
state
|
|
||||||
.video_tasks
|
|
||||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
Some(stored) => stored,
|
||||||
.await?
|
None => {
|
||||||
.ok_or_else(|| {
|
let current = match expected_user_id {
|
||||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||||
|
None => read_video_task_detail(state, task_id).await?,
|
||||||
|
};
|
||||||
|
let Some(current) = current else {
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
};
|
||||||
|
if !current.status.is_active() {
|
||||||
|
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||||
|
}
|
||||||
|
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||||
"video task repository is unavailable".to_string(),
|
"video task repository is unavailable".to_string(),
|
||||||
))
|
)));
|
||||||
})?;
|
}
|
||||||
|
};
|
||||||
|
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||||
|
state
|
||||||
|
.video_tasks
|
||||||
|
.apply_finalize_mutation(&request_path, &report_kind);
|
||||||
|
}
|
||||||
finalize_video_task_if_terminal(state, &stored).await;
|
finalize_video_task_if_terminal(state, &stored).await;
|
||||||
Ok(stored)
|
Ok(stored)
|
||||||
}
|
}
|
||||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||||
let provider_api_format = task
|
let provider_api_format = task.effective_api_format()?;
|
||||||
.provider_api_format
|
|
||||||
.as_deref()
|
|
||||||
.or(task.client_api_format.as_deref())
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|value| !value.is_empty())?;
|
|
||||||
|
|
||||||
match provider_api_format {
|
match provider_api_format {
|
||||||
"openai:video" => Some(VideoTaskCancelPlan {
|
"openai:video" => Some(VideoTaskCancelPlan {
|
||||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
|||||||
let result =
|
let result =
|
||||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|_| {
|
||||||
GatewayError::UpstreamUnavailable {
|
GatewayError::UpstreamUnavailable {
|
||||||
trace_id: trace_id.to_string(),
|
trace_id: trace_id.to_string(),
|
||||||
message: format!("{err:?}"),
|
message: "video cancellation request failed".to_string(),
|
||||||
}
|
}
|
||||||
.into_response()
|
.into_response()
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if result.status_code >= 400 {
|
if result.status_code >= 400 {
|
||||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
|
||||||
let body_json = result
|
|
||||||
.body
|
|
||||||
.and_then(|body| body.json_body)
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
json!({
|
|
||||||
"error": {
|
|
||||||
"message": result
|
|
||||||
.error
|
|
||||||
.as_ref()
|
|
||||||
.map(|error| error.message.clone())
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
format!("execution runtime returned {}", result.status_code)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
});
|
|
||||||
return Err((status, Json(body_json)).into_response());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn build_cancelled_request_metadata(
|
fn build_video_task_cancel_upstream_error_response(
|
||||||
state: &AppState,
|
result: &aether_contracts::ExecutionResult,
|
||||||
task: &StoredVideoTask,
|
) -> axum::response::Response {
|
||||||
) -> Result<Option<Value>, GatewayError> {
|
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||||
let mut metadata = match task.request_metadata.clone() {
|
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||||
Some(Value::Object(object)) => object,
|
tracing::warn!(
|
||||||
_ => Map::new(),
|
event_name = "video_task_cancel_upstream_error",
|
||||||
};
|
upstream_status = result.status_code,
|
||||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
"video cancellation upstream response body discarded"
|
||||||
if snapshot_value.is_none() {
|
);
|
||||||
snapshot_value = state
|
(
|
||||||
.reconstruct_video_task_snapshot(task)
|
status,
|
||||||
.await?
|
Json(json!({
|
||||||
.map(|snapshot| {
|
"error": {
|
||||||
serde_json::to_value(snapshot)
|
"message": format!(
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
"video cancellation upstream returned HTTP {}",
|
||||||
})
|
result.status_code
|
||||||
.transpose()?;
|
),
|
||||||
}
|
}
|
||||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
})),
|
||||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
)
|
||||||
metadata.insert(
|
.into_response()
|
||||||
"rust_owner".to_string(),
|
|
||||||
Value::String("async_task".to_string()),
|
|
||||||
);
|
|
||||||
metadata.insert(
|
|
||||||
"rust_local_snapshot".to_string(),
|
|
||||||
snapshot_value_ref.clone(),
|
|
||||||
);
|
|
||||||
return Ok(Some(Value::Object(metadata)));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(task.request_metadata.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
|
||||||
if let Some(object) = snapshot_value
|
|
||||||
.get_mut("OpenAi")
|
|
||||||
.and_then(Value::as_object_mut)
|
|
||||||
{
|
|
||||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Some(object) = snapshot_value
|
|
||||||
.get_mut("Gemini")
|
|
||||||
.and_then(Value::as_object_mut)
|
|
||||||
{
|
|
||||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn persist_cancelled_video_task(
|
async fn persist_cancelled_video_task(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
task: &StoredVideoTask,
|
task: &StoredVideoTask,
|
||||||
request_metadata: Option<Value>,
|
|
||||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||||
let now_unix_secs = current_unix_secs();
|
let now_unix_secs = current_unix_secs();
|
||||||
state
|
state
|
||||||
.data
|
.update_active_video_task(UpsertVideoTask {
|
||||||
.upsert_video_task(UpsertVideoTask {
|
|
||||||
id: task.id.clone(),
|
id: task.id.clone(),
|
||||||
short_id: task.short_id.clone(),
|
short_id: task.short_id.clone(),
|
||||||
request_id: task.request_id.clone(),
|
request_id: task.request_id.clone(),
|
||||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
|||||||
format_converted: task.format_converted,
|
format_converted: task.format_converted,
|
||||||
model: task.model.clone(),
|
model: task.model.clone(),
|
||||||
prompt: task.prompt.clone(),
|
prompt: task.prompt.clone(),
|
||||||
original_request_body: task.original_request_body.clone(),
|
original_request_body: None,
|
||||||
duration_seconds: task.duration_seconds,
|
duration_seconds: task.duration_seconds,
|
||||||
resolution: task.resolution.clone(),
|
resolution: task.resolution.clone(),
|
||||||
aspect_ratio: task.aspect_ratio.clone(),
|
aspect_ratio: task.aspect_ratio.clone(),
|
||||||
size: task.size.clone(),
|
size: task.size.clone(),
|
||||||
status: VideoTaskStatus::Cancelled,
|
status: VideoTaskStatus::Cancelled,
|
||||||
progress_percent: task.progress_percent,
|
progress_percent: task.progress_percent,
|
||||||
progress_message: task.progress_message.clone(),
|
progress_message: None,
|
||||||
retry_count: task.retry_count,
|
retry_count: task.retry_count,
|
||||||
poll_interval_seconds: task.poll_interval_seconds,
|
poll_interval_seconds: task.poll_interval_seconds,
|
||||||
next_poll_at_unix_secs: None,
|
next_poll_at_unix_secs: None,
|
||||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
|||||||
completed_at_unix_secs: Some(now_unix_secs),
|
completed_at_unix_secs: Some(now_unix_secs),
|
||||||
updated_at_unix_secs: now_unix_secs,
|
updated_at_unix_secs: now_unix_secs,
|
||||||
error_code: task.error_code.clone(),
|
error_code: task.error_code.clone(),
|
||||||
error_message: task.error_message.clone(),
|
error_message: None,
|
||||||
video_url: task.video_url.clone(),
|
video_url: task.video_url.clone(),
|
||||||
request_metadata,
|
request_metadata: None,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use aether_contracts::{
|
||||||
|
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||||
|
};
|
||||||
|
use axum::body::to_bytes;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::build_video_task_cancel_upstream_error_response;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||||
|
let result = ExecutionResult {
|
||||||
|
request_id: "cancel-secret-request-id".to_string(),
|
||||||
|
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||||
|
status_code: 502,
|
||||||
|
headers: BTreeMap::from([(
|
||||||
|
"x-internal-secret".to_string(),
|
||||||
|
"cancel-secret-header".to_string(),
|
||||||
|
)]),
|
||||||
|
response_observation: None,
|
||||||
|
body: Some(ResponseBody {
|
||||||
|
json_body: Some(json!({
|
||||||
|
"error": {
|
||||||
|
"message": "cancel-secret-upstream-body",
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
body_bytes_b64: None,
|
||||||
|
}),
|
||||||
|
telemetry: None,
|
||||||
|
error: Some(ExecutionError {
|
||||||
|
kind: ExecutionErrorKind::Upstream5xx,
|
||||||
|
phase: ExecutionPhase::FirstByte,
|
||||||
|
message: "cancel-secret-runtime-error".to_string(),
|
||||||
|
upstream_status: Some(502),
|
||||||
|
retryable: true,
|
||||||
|
failover_recommended: false,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||||
|
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||||
|
assert!(response.headers().get("x-internal-secret").is_none());
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("response body should read");
|
||||||
|
let payload: serde_json::Value =
|
||||||
|
serde_json::from_slice(&body).expect("response body should parse");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
payload,
|
||||||
|
json!({
|
||||||
|
"error": {
|
||||||
|
"message": "video cancellation upstream returned HTTP 502",
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||||
|
assert!(!body.contains("cancel-secret"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
|||||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||||
pub(crate) use http::{
|
pub(crate) use http::{
|
||||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||||
CancelVideoTaskError,
|
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||||
};
|
};
|
||||||
pub(crate) use query::{
|
pub(crate) use query::{
|
||||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||||
|
VideoTaskVideoSource,
|
||||||
};
|
};
|
||||||
pub(crate) use runtime::{
|
pub(crate) use runtime::{
|
||||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||||
|
|||||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
|||||||
pub(crate) processing_count: u64,
|
pub(crate) processing_count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub(crate) enum VideoTaskVideoSource {
|
pub(crate) enum VideoTaskVideoSource {
|
||||||
Redirect {
|
Redirect {
|
||||||
url: String,
|
url: url::Url,
|
||||||
},
|
},
|
||||||
Proxy {
|
Proxy {
|
||||||
url: String,
|
url: url::Url,
|
||||||
header_name: String,
|
header_name: String,
|
||||||
header_value: String,
|
header_value: String,
|
||||||
filename: String,
|
filename: String,
|
||||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
|||||||
state.find_video_task_by_id(task_id).await
|
state.find_video_task_by_id(task_id).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn read_video_task_detail_for_user(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||||
|
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_video_task_video_source(
|
pub(crate) async fn read_video_task_video_source(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
video_task_video_source_from_task(state, &task).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn video_task_video_source_from_task(
|
||||||
|
state: &AppState,
|
||||||
|
task: &StoredVideoTask,
|
||||||
|
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||||
let Some(video_url) = task
|
let Some(video_url) = task
|
||||||
.video_url
|
.video_url
|
||||||
.as_deref()
|
.as_deref()
|
||||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
let video_url = parse_video_url(&video_url)?;
|
||||||
|
|
||||||
|
if task.effective_api_format() != Some("gemini:video") {
|
||||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
));
|
));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||||
|
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||||
|
})?;
|
||||||
|
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||||
|
return Err(GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
let api_key = transport.key.decrypted_api_key.trim();
|
let api_key = transport.key.decrypted_api_key.trim();
|
||||||
if api_key.is_empty() {
|
if api_key.is_empty() {
|
||||||
return Err(GatewayError::Internal(
|
return Err(GatewayError::Internal(
|
||||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
url: video_url,
|
url: video_url,
|
||||||
header_name: "x-goog-api-key".to_string(),
|
header_name: "x-goog-api-key".to_string(),
|
||||||
header_value: api_key.to_string(),
|
header_value: api_key.to_string(),
|
||||||
filename: format!("video_{task_id}.mp4"),
|
filename: format!("video_{}.mp4", task.id),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||||
|
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL is invalid".to_string(),
|
||||||
|
})?;
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| url.host_str().is_none()
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
{
|
||||||
|
return Err(GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||||
|
left.scheme() == right.scheme()
|
||||||
|
&& left.host() == right.host()
|
||||||
|
&& left.port_or_known_default() == right.port_or_known_default()
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_video_task_stats(
|
pub(crate) async fn read_video_task_stats(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
filter: &VideoTaskQueryFilter,
|
filter: &VideoTaskQueryFilter,
|
||||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
|||||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||||
now_unix_secs - (now_unix_secs % 86_400)
|
now_unix_secs - (now_unix_secs % 86_400)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
|
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||||
|
use aether_data_contracts::repository::provider_catalog::{
|
||||||
|
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||||
|
StoredProviderCatalogProvider,
|
||||||
|
};
|
||||||
|
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||||
|
VideoTaskVideoSource,
|
||||||
|
};
|
||||||
|
use crate::{data::GatewayDataState, AppState};
|
||||||
|
|
||||||
|
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||||
|
{
|
||||||
|
UpsertVideoTask {
|
||||||
|
id: "legacy-gemini-task".to_string(),
|
||||||
|
short_id: Some("legacy-short".to_string()),
|
||||||
|
request_id: "legacy-request".to_string(),
|
||||||
|
user_id: Some("user-1".to_string()),
|
||||||
|
api_key_id: Some("client-key-1".to_string()),
|
||||||
|
username: None,
|
||||||
|
api_key_name: None,
|
||||||
|
external_task_id: Some("operations/upstream-1".to_string()),
|
||||||
|
provider_id: Some("provider-1".to_string()),
|
||||||
|
endpoint_id: Some("endpoint-1".to_string()),
|
||||||
|
key_id: Some("provider-key-1".to_string()),
|
||||||
|
client_api_format: Some("gemini:video".to_string()),
|
||||||
|
provider_api_format: None,
|
||||||
|
format_converted: false,
|
||||||
|
model: Some("veo-3".to_string()),
|
||||||
|
prompt: None,
|
||||||
|
original_request_body: None,
|
||||||
|
duration_seconds: Some(8),
|
||||||
|
resolution: Some("720p".to_string()),
|
||||||
|
aspect_ratio: Some("16:9".to_string()),
|
||||||
|
size: Some("1280x720".to_string()),
|
||||||
|
status: VideoTaskStatus::Completed,
|
||||||
|
progress_percent: 100,
|
||||||
|
progress_message: None,
|
||||||
|
retry_count: 0,
|
||||||
|
poll_interval_seconds: 10,
|
||||||
|
next_poll_at_unix_secs: None,
|
||||||
|
poll_count: 1,
|
||||||
|
max_poll_count: 360,
|
||||||
|
created_at_unix_ms: 1,
|
||||||
|
submitted_at_unix_secs: Some(1),
|
||||||
|
completed_at_unix_secs: Some(2),
|
||||||
|
updated_at_unix_secs: 2,
|
||||||
|
error_code: None,
|
||||||
|
error_message: None,
|
||||||
|
video_url: Some(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
request_metadata: None,
|
||||||
|
}
|
||||||
|
.into_stored()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn state_with_gemini_transport() -> AppState {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
let provider = StoredProviderCatalogProvider::new(
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"Gemini".to_string(),
|
||||||
|
Some("https://ai.google.dev".to_string()),
|
||||||
|
"gemini".to_string(),
|
||||||
|
)
|
||||||
|
.expect("provider should build");
|
||||||
|
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||||
|
"endpoint-1".to_string(),
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"gemini:video".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("endpoint should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
"https://generativelanguage.googleapis.com".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("endpoint transport should build");
|
||||||
|
let encrypted_api_key = state
|
||||||
|
.seal_provider_catalog_key_api_key(
|
||||||
|
"provider-1",
|
||||||
|
"provider-key-1",
|
||||||
|
"gemini-provider-secret",
|
||||||
|
)
|
||||||
|
.expect("provider key should encrypt");
|
||||||
|
let key = StoredProviderCatalogKey::new(
|
||||||
|
"provider-key-1".to_string(),
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"default".to_string(),
|
||||||
|
"api_key".to_string(),
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("provider key should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
Some(json!(["gemini:video"])),
|
||||||
|
encrypted_api_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("provider key transport should build");
|
||||||
|
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||||
|
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||||
|
);
|
||||||
|
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||||
|
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||||
|
video_tasks,
|
||||||
|
provider_catalog,
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
);
|
||||||
|
state.with_data_state_for_tests(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||||
|
for raw_url in [
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"data:video/mp4;base64,AAAA",
|
||||||
|
"https://[email protected]/video.mp4",
|
||||||
|
"https://user:[email protected]/video.mp4",
|
||||||
|
"/relative/video.mp4",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
parse_video_url(raw_url).is_err(),
|
||||||
|
"URL should be rejected: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||||
|
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||||
|
for same_origin in [
|
||||||
|
"https://generativelanguage.googleapis.com/file",
|
||||||
|
"https://generativelanguage.googleapis.com:443/file",
|
||||||
|
] {
|
||||||
|
assert!(video_urls_share_origin(
|
||||||
|
&base,
|
||||||
|
&parse_video_url(same_origin).unwrap()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for different_origin in [
|
||||||
|
"http://generativelanguage.googleapis.com/file",
|
||||||
|
"https://generativelanguage.googleapis.com:444/file",
|
||||||
|
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||||
|
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||||
|
] {
|
||||||
|
assert!(!video_urls_share_origin(
|
||||||
|
&base,
|
||||||
|
&parse_video_url(different_origin).unwrap()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||||
|
let source = video_task_video_source_from_task(
|
||||||
|
&state_with_gemini_transport(),
|
||||||
|
&legacy_gemini_video_task(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("video source should resolve")
|
||||||
|
.expect("video source should exist");
|
||||||
|
|
||||||
|
match source {
|
||||||
|
VideoTaskVideoSource::Proxy {
|
||||||
|
url,
|
||||||
|
header_name,
|
||||||
|
header_value,
|
||||||
|
filename,
|
||||||
|
} => {
|
||||||
|
assert_eq!(
|
||||||
|
url.as_str(),
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||||
|
);
|
||||||
|
assert_eq!(header_name, "x-goog-api-key");
|
||||||
|
assert_eq!(header_value, "gemini-provider-secret");
|
||||||
|
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||||
|
}
|
||||||
|
VideoTaskVideoSource::Redirect { .. } => {
|
||||||
|
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
|||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct VideoTaskRefreshError {
|
struct VideoTaskRefreshError {
|
||||||
message: String,
|
category: &'static str,
|
||||||
permanent: bool,
|
permanent: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
|||||||
warn!(
|
warn!(
|
||||||
event_name = "video_task_refresh_failed",
|
event_name = "video_task_refresh_failed",
|
||||||
log_type = "event",
|
log_type = "event",
|
||||||
error = %err.message,
|
error_category = err.category,
|
||||||
permanent = err.permanent,
|
permanent = err.permanent,
|
||||||
"gateway video task refresh failed"
|
"gateway video task refresh failed"
|
||||||
);
|
);
|
||||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
|||||||
let mut refreshed = 0usize;
|
let mut refreshed = 0usize;
|
||||||
for (index, task) in tasks.into_iter().enumerate() {
|
for (index, task) in tasks.into_iter().enumerate() {
|
||||||
let trace_id = format!("video-task-poller-{index}");
|
let trace_id = format!("video-task-poller-{index}");
|
||||||
|
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let Some(refresh_plan) = state
|
let Some(refresh_plan) = state
|
||||||
.video_tasks
|
.video_tasks
|
||||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||||
let Some(updated) =
|
let Some(updated) = build_successful_poll_update(
|
||||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
&task,
|
||||||
|
snapshot.clone(),
|
||||||
|
&provider_body,
|
||||||
|
now_unix_secs,
|
||||||
|
)?
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
match state.update_active_video_task(updated).await? {
|
match state.update_active_video_task(updated).await? {
|
||||||
Some(stored) => {
|
Some(stored) => {
|
||||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
if let Some(snapshot) =
|
||||||
|
state.reconstruct_video_task_snapshot(&stored).await?
|
||||||
|
{
|
||||||
state.video_tasks.record_snapshot(snapshot);
|
state.video_tasks.record_snapshot(snapshot);
|
||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
|||||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||||
match state.update_active_video_task(updated).await? {
|
match state.update_active_video_task(updated).await? {
|
||||||
Some(stored) => {
|
Some(stored) => {
|
||||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
if let Some(snapshot) =
|
||||||
|
state.reconstruct_video_task_snapshot(&stored).await?
|
||||||
|
{
|
||||||
state.video_tasks.record_snapshot(snapshot);
|
state.video_tasks.record_snapshot(snapshot);
|
||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(err) => {
|
Err(_) => {
|
||||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||||
message: format!("{err:?}"),
|
category: "transport_error",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
|||||||
.and_then(|body| body.as_object().cloned())
|
.and_then(|body| body.as_object().cloned())
|
||||||
else {
|
else {
|
||||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||||
message: "video task refresh missing json provider body".to_string(),
|
category: "invalid_provider_response",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|error| error.upstream_status)
|
.and_then(|error| error.upstream_status)
|
||||||
.unwrap_or(result.status_code);
|
.unwrap_or(result.status_code);
|
||||||
let message = result
|
let category = if status_code == 401 {
|
||||||
.error
|
"authentication_error"
|
||||||
.as_ref()
|
} else if status_code == 403 {
|
||||||
.map(|error| error.message.clone())
|
"permission_denied"
|
||||||
.or_else(|| {
|
} else if status_code == 404 {
|
||||||
result
|
"not_found"
|
||||||
.body
|
} else if status_code == 429 {
|
||||||
.as_ref()
|
"rate_limit"
|
||||||
.and_then(|body| body.json_body.as_ref())
|
} else if status_code >= 500 {
|
||||||
.and_then(|value| value.get("error"))
|
"server_error"
|
||||||
.and_then(Value::as_str)
|
} else {
|
||||||
.map(str::to_string)
|
"provider_error"
|
||||||
})
|
};
|
||||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
|
||||||
let permanent = result.error.as_ref().map_or(
|
let permanent = result.error.as_ref().map_or(
|
||||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||||
|error| match error.kind {
|
|error| match error.kind {
|
||||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
VideoTaskRefreshError { message, permanent }
|
VideoTaskRefreshError {
|
||||||
|
category,
|
||||||
|
permanent,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_successful_poll_update(
|
fn build_successful_poll_update(
|
||||||
task: &StoredVideoTask,
|
task: &StoredVideoTask,
|
||||||
|
mut snapshot: LocalVideoTaskSnapshot,
|
||||||
provider_body: &Map<String, Value>,
|
provider_body: &Map<String, Value>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
|
||||||
snapshot.apply_provider_body(provider_body);
|
snapshot.apply_provider_body(provider_body);
|
||||||
|
|
||||||
let mut record = snapshot.to_upsert_record();
|
let mut record = snapshot.to_upsert_record();
|
||||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
|||||||
record.format_converted = task.format_converted;
|
record.format_converted = task.format_converted;
|
||||||
record.model = task.model.clone().or(record.model);
|
record.model = task.model.clone().or(record.model);
|
||||||
record.prompt = task.prompt.clone().or(record.prompt);
|
record.prompt = task.prompt.clone().or(record.prompt);
|
||||||
record.original_request_body = task
|
record.original_request_body = None;
|
||||||
.original_request_body
|
|
||||||
.clone()
|
|
||||||
.or(record.original_request_body);
|
|
||||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||||
record.resolution = task.resolution.clone().or(record.resolution);
|
record.resolution = task.resolution.clone().or(record.resolution);
|
||||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
|||||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_timeout".to_string());
|
record.error_code = Some("poll_timeout".to_string());
|
||||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
}
|
}
|
||||||
record.request_metadata = merge_video_task_request_metadata(
|
record.request_metadata = None;
|
||||||
task.request_metadata.clone(),
|
|
||||||
&snapshot,
|
|
||||||
Some(provider_body),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
|
|
||||||
Ok(Some(record))
|
Ok(Some(record))
|
||||||
}
|
}
|
||||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
|||||||
let mut record = stored_task_to_upsert(task);
|
let mut record = stored_task_to_upsert(task);
|
||||||
record.updated_at_unix_secs = now_unix_secs;
|
record.updated_at_unix_secs = now_unix_secs;
|
||||||
record.poll_count = task.poll_count.saturating_add(1);
|
record.poll_count = task.poll_count.saturating_add(1);
|
||||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
record.progress_message = None;
|
||||||
if err.permanent {
|
if err.permanent {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_permanent_error".to_string());
|
record.error_code = Some("poll_permanent_error".to_string());
|
||||||
record.error_message = Some(err.message.clone());
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
} else {
|
} else {
|
||||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
|||||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_timeout".to_string());
|
record.error_code = Some("poll_timeout".to_string());
|
||||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
}
|
}
|
||||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
record.request_metadata = None;
|
||||||
.and_then(|snapshot| {
|
|
||||||
merge_video_task_request_metadata(
|
|
||||||
task.request_metadata.clone(),
|
|
||||||
&snapshot,
|
|
||||||
None,
|
|
||||||
Some(err),
|
|
||||||
)
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
})
|
|
||||||
.or(task.request_metadata.clone());
|
|
||||||
record
|
record
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||||
let snapshot_record =
|
|
||||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
|
||||||
UpsertVideoTask {
|
UpsertVideoTask {
|
||||||
id: task.id.clone(),
|
id: task.id.clone(),
|
||||||
short_id: task.short_id.clone(),
|
short_id: task.short_id.clone(),
|
||||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
|||||||
provider_api_format: task.provider_api_format.clone(),
|
provider_api_format: task.provider_api_format.clone(),
|
||||||
format_converted: task.format_converted,
|
format_converted: task.format_converted,
|
||||||
model: task.model.clone(),
|
model: task.model.clone(),
|
||||||
prompt: task.prompt.clone().or_else(|| {
|
prompt: task.prompt.clone(),
|
||||||
snapshot_record
|
original_request_body: None,
|
||||||
.as_ref()
|
duration_seconds: task.duration_seconds,
|
||||||
.and_then(|record| record.prompt.clone())
|
resolution: task.resolution.clone(),
|
||||||
}),
|
aspect_ratio: task.aspect_ratio.clone(),
|
||||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
size: task.size.clone(),
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.original_request_body.clone())
|
|
||||||
}),
|
|
||||||
duration_seconds: task.duration_seconds.or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.duration_seconds)
|
|
||||||
}),
|
|
||||||
resolution: task.resolution.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.resolution.clone())
|
|
||||||
}),
|
|
||||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.aspect_ratio.clone())
|
|
||||||
}),
|
|
||||||
size: task.size.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.size.clone())
|
|
||||||
}),
|
|
||||||
status: task.status,
|
status: task.status,
|
||||||
progress_percent: task.progress_percent,
|
progress_percent: task.progress_percent,
|
||||||
progress_message: task.progress_message.clone(),
|
progress_message: None,
|
||||||
retry_count: task.retry_count,
|
retry_count: task.retry_count,
|
||||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
|||||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||||
error_code: task.error_code.clone(),
|
error_code: task.error_code.clone(),
|
||||||
error_message: task.error_message.clone(),
|
error_message: None,
|
||||||
video_url: task.video_url.clone(),
|
video_url: task.video_url.clone(),
|
||||||
request_metadata: task.request_metadata.clone(),
|
request_metadata: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
|||||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_video_task_request_metadata(
|
|
||||||
existing: Option<Value>,
|
|
||||||
snapshot: &LocalVideoTaskSnapshot,
|
|
||||||
provider_body: Option<&Map<String, Value>>,
|
|
||||||
poll_error: Option<&VideoTaskRefreshError>,
|
|
||||||
) -> Result<Option<Value>, serde_json::Error> {
|
|
||||||
let mut metadata = match existing {
|
|
||||||
Some(Value::Object(object)) => object,
|
|
||||||
_ => Map::new(),
|
|
||||||
};
|
|
||||||
metadata.insert(
|
|
||||||
"rust_owner".to_string(),
|
|
||||||
Value::String("async_task".to_string()),
|
|
||||||
);
|
|
||||||
metadata.insert(
|
|
||||||
"rust_local_snapshot".to_string(),
|
|
||||||
serde_json::to_value(snapshot)?,
|
|
||||||
);
|
|
||||||
if let Some(provider_body) = provider_body {
|
|
||||||
metadata.insert(
|
|
||||||
"poll_raw_response".to_string(),
|
|
||||||
Value::Object(provider_body.clone()),
|
|
||||||
);
|
|
||||||
metadata.remove("poll_error");
|
|
||||||
}
|
|
||||||
if let Some(poll_error) = poll_error {
|
|
||||||
metadata.insert(
|
|
||||||
"poll_error".to_string(),
|
|
||||||
serde_json::json!({
|
|
||||||
"message": poll_error.message,
|
|
||||||
"permanent": poll_error.permanent,
|
|
||||||
"observed_at_unix_secs": now_unix_secs(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(Some(Value::Object(metadata)))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||||
return;
|
return;
|
||||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
let provider_name = task
|
||||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
.provider_id
|
||||||
.or_else(|| task.provider_id.clone())
|
.clone()
|
||||||
.unwrap_or_else(|| "unknown".to_string());
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
let response_time_ms = task
|
let response_time_ms = task
|
||||||
.submitted_at_unix_secs
|
.submitted_at_unix_secs
|
||||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
|||||||
has_format_conversion: Some(task.format_converted),
|
has_format_conversion: Some(task.format_converted),
|
||||||
is_stream: Some(false),
|
is_stream: Some(false),
|
||||||
status_code,
|
status_code,
|
||||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
error_message: task.error_code.clone(),
|
||||||
response_time_ms,
|
response_time_ms,
|
||||||
request_body: task.original_request_body.clone(),
|
request_body: None,
|
||||||
request_metadata: task.request_metadata.clone(),
|
request_metadata: None,
|
||||||
..UsageEventData::default()
|
..UsageEventData::default()
|
||||||
},
|
},
|
||||||
))
|
))
|
||||||
@@ -609,6 +536,9 @@ mod tests {
|
|||||||
|
|
||||||
fn sample_sparse_stored_task() -> StoredVideoTask {
|
fn sample_sparse_stored_task() -> StoredVideoTask {
|
||||||
let snapshot = LocalVideoTaskSnapshot::OpenAi(OpenAiVideoTaskSeed {
|
let snapshot = LocalVideoTaskSnapshot::OpenAi(OpenAiVideoTaskSeed {
|
||||||
|
local_short_id: None,
|
||||||
|
native_response: None,
|
||||||
|
xai_provider: false,
|
||||||
local_task_id: "task-1".to_string(),
|
local_task_id: "task-1".to_string(),
|
||||||
upstream_task_id: "ext-1".to_string(),
|
upstream_task_id: "ext-1".to_string(),
|
||||||
created_at_unix_ms: 1,
|
created_at_unix_ms: 1,
|
||||||
@@ -701,48 +631,36 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||||
|
|
||||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
assert!(record.prompt.is_none());
|
||||||
assert_eq!(
|
assert!(record.original_request_body.is_none());
|
||||||
record.original_request_body,
|
assert!(record.duration_seconds.is_none());
|
||||||
Some(json!({
|
assert!(record.resolution.is_none());
|
||||||
"prompt": "hello",
|
assert!(record.aspect_ratio.is_none());
|
||||||
"seconds": "4",
|
assert!(record.size.is_none());
|
||||||
"resolution": "720p",
|
assert!(record.progress_message.is_none());
|
||||||
"aspect_ratio": "16:9",
|
assert!(record.error_message.is_none());
|
||||||
"size": "1280x720"
|
assert!(record.request_metadata.is_none());
|
||||||
}))
|
|
||||||
);
|
|
||||||
assert_eq!(record.duration_seconds, Some(4));
|
|
||||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
|
||||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
|
||||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||||
let record = build_failed_poll_update(
|
let record = build_failed_poll_update(
|
||||||
&sample_sparse_stored_task(),
|
&sample_sparse_stored_task(),
|
||||||
&VideoTaskRefreshError {
|
&VideoTaskRefreshError {
|
||||||
message: "temporary failure".to_string(),
|
category: "transport_error",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
},
|
},
|
||||||
100,
|
100,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(
|
assert!(record.original_request_body.is_none());
|
||||||
record.original_request_body,
|
assert!(record.prompt.is_none());
|
||||||
Some(json!({
|
assert!(record.resolution.is_none());
|
||||||
"prompt": "hello",
|
assert!(record.progress_message.is_none());
|
||||||
"seconds": "4",
|
assert!(record.error_message.is_none());
|
||||||
"resolution": "720p",
|
assert!(record.request_metadata.is_none());
|
||||||
"aspect_ratio": "16:9",
|
|
||||||
"size": "1280x720"
|
|
||||||
}))
|
|
||||||
);
|
|
||||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
|
||||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
path_and_query: &str,
|
path_and_query: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
) {
|
) {
|
||||||
|
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||||
let Some(decision) = control_decision else {
|
let Some(decision) = control_decision else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
|||||||
},
|
},
|
||||||
route_kind,
|
route_kind,
|
||||||
default_target_type(route_family),
|
default_target_type(route_family),
|
||||||
path_and_query.to_string(),
|
sanitized_path_and_query.clone(),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||||
|
|
||||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||||
if log_level == AdminAuditLogLevel::Info {
|
if log_level == AdminAuditLogLevel::Info {
|
||||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
route_family,
|
route_family,
|
||||||
route_kind,
|
route_kind,
|
||||||
method = %method,
|
method = %method,
|
||||||
path = %path_and_query,
|
path = %sanitized_path_and_query,
|
||||||
action,
|
action,
|
||||||
target_type,
|
target_type,
|
||||||
target_id = %target_id,
|
target_id = %target_id,
|
||||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
route_family,
|
route_family,
|
||||||
route_kind,
|
route_kind,
|
||||||
method = %method,
|
method = %method,
|
||||||
path = %path_and_query,
|
path = %sanitized_path_and_query,
|
||||||
action,
|
action,
|
||||||
target_type,
|
target_type,
|
||||||
target_id = %target_id,
|
target_id = %target_id,
|
||||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||||
|
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||||
|
if target_id.trim_start().starts_with('/') {
|
||||||
|
return sanitize_admin_audit_path(&target_id);
|
||||||
|
}
|
||||||
|
target_id
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
enum AdminAuditLogLevel {
|
enum AdminAuditLogLevel {
|
||||||
Info,
|
Info,
|
||||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
use super::{
|
||||||
|
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||||
|
AdminAuditLogLevel,
|
||||||
|
};
|
||||||
use axum::http::{Method, StatusCode};
|
use axum::http::{Method, StatusCode};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -169,4 +185,32 @@ mod tests {
|
|||||||
("failed", AdminAuditLogLevel::Warn)
|
("failed", AdminAuditLogLevel::Warn)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn audit_paths_drop_sensitive_query_values() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_path(
|
||||||
|
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||||
|
),
|
||||||
|
"/api/admin/providers?limit=25"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||||
|
"/install/[redacted]?view=raw"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_target_id(
|
||||||
|
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||||
|
),
|
||||||
|
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||||
|
"resource-id?literal"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||||
|
|
||||||
match trace {
|
match trace {
|
||||||
Some(trace) => Ok(Json(trace)),
|
Some(mut trace) => {
|
||||||
|
trace.sanitize_sensitive_diagnostics();
|
||||||
|
Ok(Json(trace))
|
||||||
|
}
|
||||||
None => Err((
|
None => Err((
|
||||||
axum::http::StatusCode::NOT_FOUND,
|
axum::http::StatusCode::NOT_FOUND,
|
||||||
Json(json!({
|
Json(json!({
|
||||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||||
|
|
||||||
match trace {
|
match trace {
|
||||||
Some(trace) => Ok(Json(trace)),
|
Some(mut trace) => {
|
||||||
|
trace.sanitize_sensitive_diagnostics();
|
||||||
|
Ok(Json(trace))
|
||||||
|
}
|
||||||
None => Err((
|
None => Err((
|
||||||
axum::http::StatusCode::NOT_FOUND,
|
axum::http::StatusCode::NOT_FOUND,
|
||||||
Json(json!({
|
Json(json!({
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
|||||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||||
use super::scopes::BackupScope;
|
use super::scopes::BackupScope;
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct S3BackupConfig {
|
pub(crate) struct S3BackupConfig {
|
||||||
pub(crate) enabled: bool,
|
pub(crate) enabled: bool,
|
||||||
pub(crate) scope: BackupScope,
|
pub(crate) scope: BackupScope,
|
||||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
|||||||
pub(crate) retention_count: u32,
|
pub(crate) retention_count: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for S3BackupConfig {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||||
|
formatter
|
||||||
|
.debug_struct("S3BackupConfig")
|
||||||
|
.field("enabled", &self.enabled)
|
||||||
|
.field("scope", &self.scope)
|
||||||
|
.field("endpoint_origin", &endpoint_origin)
|
||||||
|
.field("region", &self.region)
|
||||||
|
.field("user_agent", &self.user_agent)
|
||||||
|
.field("bucket", &self.bucket)
|
||||||
|
.field("prefix", &self.prefix)
|
||||||
|
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||||
|
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||||
|
.field("path_style", &self.path_style)
|
||||||
|
.field("compression", &self.compression)
|
||||||
|
.field("schedule", &self.schedule)
|
||||||
|
.field("retention_count", &self.retention_count)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct BackupConfigError {
|
pub(crate) struct BackupConfigError {
|
||||||
message: String,
|
message: String,
|
||||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
|||||||
"Endpoint(S3 地址)",
|
"Endpoint(S3 地址)",
|
||||||
enabled,
|
enabled,
|
||||||
)?;
|
)?;
|
||||||
|
if enabled {
|
||||||
|
validate_s3_endpoint(&endpoint)?;
|
||||||
|
}
|
||||||
let bucket =
|
let bucket =
|
||||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||||
let access_key_id = required_or_disabled_string(
|
let access_key_id = required_or_disabled_string(
|
||||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
|||||||
enabled,
|
enabled,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
let prefix = normalize_s3_prefix(
|
||||||
|
&optional_string(entries, "backup_s3_prefix")?
|
||||||
|
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||||
|
)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
enabled,
|
enabled,
|
||||||
scope,
|
scope,
|
||||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
|||||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||||
bucket,
|
bucket,
|
||||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
prefix,
|
||||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
|
||||||
access_key_id,
|
access_key_id,
|
||||||
secret_access_key,
|
secret_access_key,
|
||||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||||
|
let prefix = prefix.trim().trim_matches('/');
|
||||||
|
if prefix.is_empty() {
|
||||||
|
return Ok(String::new());
|
||||||
|
}
|
||||||
|
if prefix
|
||||||
|
.split('/')
|
||||||
|
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||||
|
|| prefix.contains('\\')
|
||||||
|
{
|
||||||
|
return Err(BackupConfigError::new(
|
||||||
|
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(format!("{prefix}/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||||
|
let parsed = url::Url::parse(endpoint)
|
||||||
|
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||||
|
if parsed.scheme() != "https"
|
||||||
|
|| parsed.host_str().is_none()
|
||||||
|
|| !parsed.username().is_empty()
|
||||||
|
|| parsed.password().is_some()
|
||||||
|
|| parsed.query().is_some()
|
||||||
|
|| parsed.fragment().is_some()
|
||||||
|
{
|
||||||
|
return Err(BackupConfigError::new(
|
||||||
|
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||||
|
url::Url::parse(endpoint)
|
||||||
|
.ok()
|
||||||
|
.map(|parsed| parsed.origin().ascii_serialization())
|
||||||
|
.unwrap_or_else(|| "<invalid>".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||||
if (min..=max).contains(&value) {
|
if (min..=max).contains(&value) {
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -374,6 +445,69 @@ mod tests {
|
|||||||
assert!(err.to_string().contains("Endpoint"));
|
assert!(err.to_string().contains("Endpoint"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||||
|
for endpoint in [
|
||||||
|
"http://s3.example.com",
|
||||||
|
"https://user:[email protected]",
|
||||||
|
"https://s3.example.com?token=secret",
|
||||||
|
"https://s3.example.com/#fragment",
|
||||||
|
] {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": endpoint,
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_access_key_id": "access",
|
||||||
|
"backup_s3_secret_access_key": "secret"
|
||||||
|
});
|
||||||
|
|
||||||
|
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect_err("unsafe endpoint should fail closed");
|
||||||
|
assert!(error.to_string().contains("Endpoint"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debug_output_does_not_expose_s3_credentials() {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_access_key_id": "access-key-value",
|
||||||
|
"backup_s3_secret_access_key": "secret-key-value"
|
||||||
|
});
|
||||||
|
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect("config should parse");
|
||||||
|
|
||||||
|
let debug = format!("{config:?}");
|
||||||
|
assert!(debug.contains("https://s3.example.com"));
|
||||||
|
assert!(!debug.contains("/path"));
|
||||||
|
assert!(!debug.contains("access-key-value"));
|
||||||
|
assert!(!debug.contains("secret-key-value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonicalizes_s3_backup_prefix_once() {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": "https://s3.example.com",
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_prefix": "/prod/backups//",
|
||||||
|
"backup_s3_access_key_id": "access",
|
||||||
|
"backup_s3_secret_access_key": "secret"
|
||||||
|
});
|
||||||
|
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect("prefix should be canonicalized");
|
||||||
|
|
||||||
|
assert_eq!(config.prefix, "prod/backups/");
|
||||||
|
|
||||||
|
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||||
|
let mut entries = entries.clone();
|
||||||
|
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||||
|
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn applies_default_values_from_system_config_contract() {
|
fn applies_default_values_from_system_config_contract() {
|
||||||
let entries = serde_json::json!({
|
let entries = serde_json::json!({
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
|||||||
pub(crate) mod task;
|
pub(crate) mod task;
|
||||||
pub(crate) mod worker;
|
pub(crate) mod worker;
|
||||||
|
|
||||||
|
pub use executor::{
|
||||||
|
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||||
|
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||||
|
};
|
||||||
|
|
||||||
|
use axum::body::Bytes;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum BackupRestoreScope {
|
||||||
|
Config,
|
||||||
|
Users,
|
||||||
|
Data,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupRestoreScope {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Config => "config",
|
||||||
|
Self::Users => "users",
|
||||||
|
Self::Data => "data",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
#[error("backup database apply failed: {0}")]
|
||||||
|
pub struct BackupApplyError(String);
|
||||||
|
|
||||||
|
pub async fn apply_restored_backup(
|
||||||
|
app: &crate::AppState,
|
||||||
|
restored: RestoredBackupJson,
|
||||||
|
scope: BackupRestoreScope,
|
||||||
|
operator_id: Option<&str>,
|
||||||
|
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||||
|
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||||
|
if authority.scope() != scope {
|
||||||
|
return Err(BackupApplyError(format!(
|
||||||
|
"authenticated {} backup cannot be applied to {} scope",
|
||||||
|
authority.scope().as_str(),
|
||||||
|
scope.as_str(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let request_body = Bytes::from(json_bytes);
|
||||||
|
let state = crate::admin_api::AdminAppState::new(app);
|
||||||
|
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||||
|
match scope {
|
||||||
|
BackupRestoreScope::Config => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_config_backup(&request_body, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupRestoreScope::Users => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupRestoreScope::Data => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
let message = match error {
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||||
|
"another system import or restore is already running"
|
||||||
|
}
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||||
|
"system import coordination is unavailable"
|
||||||
|
}
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||||
|
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
BackupApplyError(message.to_string())
|
||||||
|
})?;
|
||||||
|
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{
|
||||||
|
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||||
|
RestoredBackupJson,
|
||||||
|
};
|
||||||
|
use crate::backup::executor::encrypt_backup_bytes;
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||||
|
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||||
|
let compressed = zstd::stream::encode_all(
|
||||||
|
serde_json::to_vec(&json!({
|
||||||
|
"version": "1.5",
|
||||||
|
"exported_at": "2026-08-30T12:00:00Z",
|
||||||
|
"users": [],
|
||||||
|
"standalone_keys": [],
|
||||||
|
}))
|
||||||
|
.expect("test backup should serialize")
|
||||||
|
.as_slice(),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.expect("test backup should compress");
|
||||||
|
let (envelope, _) =
|
||||||
|
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||||
|
.expect("test backup should encrypt");
|
||||||
|
super::restore_backup_json(
|
||||||
|
object_key,
|
||||||
|
&envelope,
|
||||||
|
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.expect("test restore key should build")],
|
||||||
|
BackupRestoreLimits::default(),
|
||||||
|
)
|
||||||
|
.expect("test backup should authenticate")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||||
|
let restored = authenticated_users_backup();
|
||||||
|
|
||||||
|
let error = apply_restored_backup(
|
||||||
|
&crate::AppState::new().expect("test state should build"),
|
||||||
|
restored,
|
||||||
|
BackupRestoreScope::Config,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("scope mismatch must fail before database access");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.to_string(),
|
||||||
|
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||||
|
let app = crate::AppState::new().expect("test state should build");
|
||||||
|
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||||
|
.await
|
||||||
|
.expect("test should acquire the shared import lease");
|
||||||
|
|
||||||
|
let error = apply_restored_backup(
|
||||||
|
&app,
|
||||||
|
authenticated_users_backup(),
|
||||||
|
BackupRestoreScope::Users,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("restore must not interleave with another system import");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.to_string(),
|
||||||
|
"backup database apply failed: another system import or restore is already running"
|
||||||
|
);
|
||||||
|
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
|
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||||
|
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub(crate) enum BackupScope {
|
pub(crate) enum BackupScope {
|
||||||
Config,
|
Config,
|
||||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||||
|
if object_key.is_empty()
|
||||||
|
|| object_key.starts_with('/')
|
||||||
|
|| object_key.contains('\0')
|
||||||
|
|| object_key.contains('\\')
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut segments = object_key.split('/').peekable();
|
||||||
|
let mut file_name = None;
|
||||||
|
while let Some(segment) = segments.next() {
|
||||||
|
if segment.is_empty()
|
||||||
|
|| segment == "."
|
||||||
|
|| segment == ".."
|
||||||
|
|| segment.chars().any(char::is_control)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if segments.peek().is_none() {
|
||||||
|
file_name = Some(segment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let file_name = file_name?;
|
||||||
|
|
||||||
|
[Self::Config, Self::Users, Self::Data]
|
||||||
|
.into_iter()
|
||||||
|
.find(|scope| {
|
||||||
|
file_name
|
||||||
|
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||||
|
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||||
|
.is_some_and(is_aether_backup_object_id)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn matching_backup_keys(
|
pub(crate) fn matching_backup_keys(
|
||||||
self,
|
self,
|
||||||
prefix: &str,
|
prefix: &str,
|
||||||
keys: impl IntoIterator<Item = String>,
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(
|
||||||
|
prefix,
|
||||||
|
keys,
|
||||||
|
&[
|
||||||
|
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||||
|
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn matching_encrypted_backup_keys(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(
|
||||||
|
prefix,
|
||||||
|
keys,
|
||||||
|
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matching_backup_keys_with_suffixes(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
file_suffixes: &[&str],
|
||||||
) -> Vec<String> {
|
) -> Vec<String> {
|
||||||
let normalized_prefix = normalized_prefix(prefix);
|
let normalized_prefix = normalized_prefix(prefix);
|
||||||
let expected_prefix = if normalized_prefix.is_empty() {
|
let expected_prefix = if normalized_prefix.is_empty() {
|
||||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
|||||||
format!("{normalized_prefix}/")
|
format!("{normalized_prefix}/")
|
||||||
};
|
};
|
||||||
let file_prefix = format!("{}-", self.file_stem());
|
let file_prefix = format!("{}-", self.file_stem());
|
||||||
let file_suffix = ".json.zst";
|
|
||||||
|
|
||||||
keys.into_iter()
|
keys.into_iter()
|
||||||
.filter(|key| {
|
.filter(|key| {
|
||||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
|||||||
if file_name.contains('/') {
|
if file_name.contains('/') {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
let Some(timestamp) = file_name
|
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||||
.strip_prefix(&file_prefix)
|
file_suffixes
|
||||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
.iter()
|
||||||
else {
|
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||||
|
}) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
is_aether_backup_timestamp(timestamp)
|
is_aether_backup_object_id(timestamp)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn file_name(self, timestamp: &str) -> String {
|
fn file_name(self, timestamp: &str) -> String {
|
||||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
format!(
|
||||||
|
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||||
|
self.file_stem()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
|||||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||||
|
if is_aether_backup_timestamp(value) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||||
|
let (time, digest) = rest.split_once('-')?;
|
||||||
|
Some((format!("{date}-{time}"), digest))
|
||||||
|
}) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
is_aether_backup_timestamp(×tamp)
|
||||||
|
&& collision_digest.len() == 64
|
||||||
|
&& collision_digest
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::BackupScope;
|
use super::BackupScope;
|
||||||
@@ -130,15 +226,15 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,7 +242,7 @@ mod tests {
|
|||||||
fn retention_filter_only_matches_same_scope() {
|
fn retention_filter_only_matches_same_scope() {
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||||
"prod/random.json.zst".to_string(),
|
"prod/random.json.zst".to_string(),
|
||||||
];
|
];
|
||||||
@@ -155,14 +251,18 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
matched,
|
matched,
|
||||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn retention_filter_requires_aether_timestamp_format() {
|
fn retention_filter_requires_aether_timestamp_format() {
|
||||||
|
let collision_digest = "a".repeat(64);
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||||
|
format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||||
@@ -171,13 +271,19 @@ mod tests {
|
|||||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||||
|
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||||
];
|
];
|
||||||
|
|
||||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
matched,
|
matched,
|
||||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
vec![
|
||||||
|
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||||
|
format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,11 +291,11 @@ mod tests {
|
|||||||
fn backup_key_prefix_boundaries_are_exact() {
|
fn backup_key_prefix_boundaries_are_exact() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("", "20260524-031500"),
|
BackupScope::Config.object_key("", "20260524-031500"),
|
||||||
"aether-config-backup-20260524-031500.json.zst"
|
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
|
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
@@ -208,4 +314,36 @@ mod tests {
|
|||||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||||
|
let collision_digest = "a".repeat(64);
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(
|
||||||
|
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
|
Some(BackupScope::Config)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(&format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
)),
|
||||||
|
Some(BackupScope::Users)
|
||||||
|
);
|
||||||
|
for key in [
|
||||||
|
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||||
|
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(key),
|
||||||
|
None,
|
||||||
|
"unsafe or unrelated key: {key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::{Bytes, BytesMut};
|
||||||
use futures_util::TryStreamExt;
|
use futures_util::TryStreamExt;
|
||||||
use object_store::aws::AmazonS3Builder;
|
use object_store::aws::AmazonS3Builder;
|
||||||
use object_store::path::Path;
|
use object_store::path::Path;
|
||||||
use object_store::{ClientOptions, ObjectStore};
|
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||||
use reqwest::header::HeaderValue;
|
use reqwest::header::HeaderValue;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
use super::config::S3BackupConfig;
|
use super::config::S3BackupConfig;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum BackupObjectCreateResult {
|
||||||
|
Created,
|
||||||
|
AlreadyExists,
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
async fn put_object_if_absent(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError>;
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
async fn put_object_if_absent(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||||
|
let mut objects = self.objects.write().await;
|
||||||
|
if objects.contains_key(key) {
|
||||||
|
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||||
|
} else {
|
||||||
|
objects.insert(key.to_string(), bytes);
|
||||||
|
Ok(BackupObjectCreateResult::Created)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError> {
|
||||||
|
let bytes = self
|
||||||
|
.objects
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(key)
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||||
|
if bytes.len() > max_bytes {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||||
|
self.objects.write().await.remove(key);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError> {
|
||||||
let prefix = directory_list_prefix(prefix);
|
let prefix = directory_list_prefix(prefix);
|
||||||
Ok(self
|
let keys: Vec<_> = self
|
||||||
.objects
|
.objects
|
||||||
.read()
|
.read()
|
||||||
.await
|
.await
|
||||||
.keys()
|
.keys()
|
||||||
.filter(|key| key.starts_with(&prefix))
|
.filter(|key| key.starts_with(&prefix))
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect())
|
.collect();
|
||||||
|
if keys.len() > max_objects {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object listing exceeds the configured {max_objects} object limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(keys)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
#[cfg(test)]
|
||||||
self.objects.write().await.remove(key);
|
impl FakeBackupObjectStore {
|
||||||
Ok(())
|
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||||
|
self.objects.read().await.get(key).cloned()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
|||||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
async fn put_object_if_absent(
|
||||||
let prefix_path = list_prefix_path(prefix);
|
&self,
|
||||||
let mut keys = self
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||||
|
let options = PutOptions {
|
||||||
|
mode: PutMode::Create,
|
||||||
|
..PutOptions::default()
|
||||||
|
};
|
||||||
|
match self
|
||||||
.store
|
.store
|
||||||
.list(prefix_path.as_ref())
|
.put_opts(&Path::from(key), bytes.into(), options)
|
||||||
.map_ok(|meta| meta.location.to_string())
|
|
||||||
.try_collect::<Vec<_>>()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
{
|
||||||
keys.sort();
|
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||||
Ok(keys)
|
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||||
|
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||||
|
}
|
||||||
|
Err(error) => Err(BackupStoreError::object_store(
|
||||||
|
"conditional put",
|
||||||
|
key,
|
||||||
|
error,
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError> {
|
||||||
|
let result = self
|
||||||
|
.store
|
||||||
|
.get(&Path::from(key))
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||||
|
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let object_size = result.meta.size;
|
||||||
|
let mut stream = result.into_stream();
|
||||||
|
let mut bytes = BytesMut::with_capacity(
|
||||||
|
usize::try_from(object_size)
|
||||||
|
.unwrap_or(max_bytes)
|
||||||
|
.min(max_bytes)
|
||||||
|
.min(8 * 1024 * 1024),
|
||||||
|
);
|
||||||
|
while let Some(chunk) = stream
|
||||||
|
.try_next()
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||||
|
{
|
||||||
|
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&chunk);
|
||||||
|
}
|
||||||
|
Ok(bytes.freeze())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError> {
|
||||||
|
let prefix_path = list_prefix_path(prefix);
|
||||||
|
let mut objects = self.store.list(prefix_path.as_ref());
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
while let Some(meta) = objects
|
||||||
|
.try_next()
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||||
|
{
|
||||||
|
if keys.len() >= max_objects {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object listing exceeds the configured {max_objects} object limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
keys.push(meta.location.to_string());
|
||||||
|
}
|
||||||
|
keys.sort();
|
||||||
|
Ok(keys)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn directory_list_prefix(prefix: &str) -> String {
|
fn directory_list_prefix(prefix: &str) -> String {
|
||||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
use super::{
|
||||||
|
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||||
|
};
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
async fn fake_backup_object_store_puts_and_lists() {
|
||||||
let store = FakeBackupObjectStore::default();
|
let store = FakeBackupObjectStore::default();
|
||||||
store
|
store
|
||||||
.put_object(
|
.put_object(
|
||||||
@@ -186,17 +336,59 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let keys = store.list_keys("prod/").await.unwrap();
|
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||||
assert_eq!(keys.len(), 2);
|
|
||||||
|
|
||||||
store
|
|
||||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let keys = store.list_keys("prod/").await.unwrap();
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
keys,
|
keys,
|
||||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
vec![
|
||||||
|
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||||
|
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||||
|
let store = FakeBackupObjectStore::default();
|
||||||
|
store
|
||||||
|
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||||
|
bytes::Bytes::from_static(b"1234")
|
||||||
|
);
|
||||||
|
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||||
|
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||||
|
let store = FakeBackupObjectStore::default();
|
||||||
|
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
BackupObjectCreateResult::Created
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
BackupObjectCreateResult::AlreadyExists
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.object_bytes(key).await.as_deref(),
|
||||||
|
Some(b"first".as_slice())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,7 +410,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let keys = store.list_keys("prod").await.unwrap();
|
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
keys,
|
keys,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
use std::future::Future;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use aether_admin::system::admin_system_config_default_value;
|
use aether_admin::system::admin_system_config_default_value;
|
||||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
|||||||
use futures_util::FutureExt;
|
use futures_util::FutureExt;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
use tokio::task::{JoinError, JoinHandle};
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use super::config::S3BackupConfig;
|
use super::config::S3BackupConfig;
|
||||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||||
use super::scopes::BackupScope;
|
use super::scopes::BackupScope;
|
||||||
use super::store::ObjectStoreS3BackupStore;
|
use super::store::ObjectStoreS3BackupStore;
|
||||||
use crate::admin_api::AdminAppState;
|
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||||
use crate::task_runtime::{
|
use crate::task_runtime::{
|
||||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
|||||||
];
|
];
|
||||||
|
|
||||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||||
|
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||||
|
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||||
|
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
|||||||
detail: String,
|
detail: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum BackupLockRenewalFailure<E> {
|
||||||
|
Lost,
|
||||||
|
Backend(E),
|
||||||
|
}
|
||||||
|
|
||||||
|
enum BackupLockRaceOutcome<T> {
|
||||||
|
BackupCompleted(T),
|
||||||
|
LeaseLost(Result<(), JoinError>),
|
||||||
|
}
|
||||||
|
|
||||||
impl S3BackupTaskError {
|
impl S3BackupTaskError {
|
||||||
fn bad_request(detail: impl Into<String>) -> Self {
|
fn bad_request(detail: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
|||||||
impl std::error::Error for S3BackupTaskError {}
|
impl std::error::Error for S3BackupTaskError {}
|
||||||
|
|
||||||
impl From<GatewayError> for S3BackupTaskError {
|
impl From<GatewayError> for S3BackupTaskError {
|
||||||
fn from(error: GatewayError) -> Self {
|
fn from(_error: GatewayError) -> Self {
|
||||||
Self::internal(format!("{error:?}"))
|
warn!(
|
||||||
|
error_category = "dependency_failed",
|
||||||
|
"S3 backup dependency failed"
|
||||||
|
);
|
||||||
|
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
|||||||
) -> Value {
|
) -> Value {
|
||||||
let mut payload = json!({
|
let mut payload = json!({
|
||||||
"scope": config.scope.as_config_value(),
|
"scope": config.scope.as_config_value(),
|
||||||
"bucket": config.bucket.clone(),
|
|
||||||
"prefix": config.prefix.clone(),
|
|
||||||
"compression": config.compression.clone(),
|
"compression": config.compression.clone(),
|
||||||
"trigger": trigger,
|
"trigger": trigger,
|
||||||
});
|
});
|
||||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务异常退出".to_string()),
|
Some("S3 备份任务异常退出".to_string()),
|
||||||
None,
|
None,
|
||||||
Some("S3 backup task panicked".to_string()),
|
Some("background_task_panicked".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
|||||||
.await;
|
.await;
|
||||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||||
|
|
||||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||||
let result = run_s3_backup_once(&app, &config).await;
|
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||||
heartbeat.abort();
|
.await
|
||||||
let _ = heartbeat.await;
|
{
|
||||||
|
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||||
|
match require_successful_backup_lock_renewal(
|
||||||
|
app.runtime_state
|
||||||
|
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||||
|
.await,
|
||||||
|
) {
|
||||||
|
Ok(()) => result,
|
||||||
|
Err(BackupLockRenewalFailure::Lost) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
"S3 backup task lost its distributed lock before publishing completion"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||||
|
match heartbeat_result {
|
||||||
|
Ok(()) => warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup task stopped after losing its distributed lock"
|
||||||
|
),
|
||||||
|
Err(error) => warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup lock heartbeat task failed"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"S3 备份任务锁已失效,任务已停止",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
warn!(
|
||||||
|
error_category = "slot_record_failed",
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup slot record failed"
|
||||||
|
);
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
&run_id,
|
&run_id,
|
||||||
"failed",
|
"failed",
|
||||||
"S3 backup slot record failed",
|
"S3 backup slot record failed",
|
||||||
Some(json!({ "error": format!("{error:?}") })),
|
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return;
|
return;
|
||||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
warn!(
|
||||||
|
error_category = "backup_execution_failed",
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup task failed"
|
||||||
|
);
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务失败".to_string()),
|
Some("S3 备份任务失败".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(error.to_string()),
|
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
|||||||
&run_id,
|
&run_id,
|
||||||
"failed",
|
"failed",
|
||||||
"S3 backup task failed",
|
"S3 backup task failed",
|
||||||
Some(json!({ "error": error.to_string() })),
|
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||||
|
backup: F,
|
||||||
|
mut heartbeat: JoinHandle<()>,
|
||||||
|
) -> BackupLockRaceOutcome<T>
|
||||||
|
where
|
||||||
|
F: Future<Output = T>,
|
||||||
|
{
|
||||||
|
tokio::pin!(backup);
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
heartbeat_result = &mut heartbeat => {
|
||||||
|
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||||
|
}
|
||||||
|
result = &mut backup => {
|
||||||
|
heartbeat.abort();
|
||||||
|
let _ = heartbeat.await;
|
||||||
|
BackupLockRaceOutcome::BackupCompleted(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_successful_backup_lock_renewal<E>(
|
||||||
|
result: Result<bool, E>,
|
||||||
|
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||||
|
match result {
|
||||||
|
Ok(true) => Ok(()),
|
||||||
|
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||||
|
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn spawn_s3_backup_task_heartbeat(
|
fn spawn_s3_backup_task_heartbeat(
|
||||||
app: AppState,
|
app: AppState,
|
||||||
run_id: String,
|
run_id: String,
|
||||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
|||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
let _ = app
|
match require_successful_backup_lock_renewal(
|
||||||
.runtime_state
|
app.runtime_state
|
||||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||||
.await;
|
.await,
|
||||||
|
) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(BackupLockRenewalFailure::Lost) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
"S3 backup task distributed lock is no longer owned"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup task distributed lock renewal failed"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
|||||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||||
)),
|
)),
|
||||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
Err(_) => {
|
||||||
"无法获取 S3 备份任务锁:{error}"
|
warn!(
|
||||||
))),
|
error_category = "lock_acquisition_failed",
|
||||||
|
"S3 backup task lock acquisition failed"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"无法获取 S3 备份任务锁,请稍后重试",
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
|||||||
app: &AppState,
|
app: &AppState,
|
||||||
config: &S3BackupConfig,
|
config: &S3BackupConfig,
|
||||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||||
let admin_state = AdminAppState::new(app);
|
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||||
let payload = match config.scope {
|
return Err(S3BackupTaskError::service_unavailable(
|
||||||
BackupScope::Config => {
|
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||||
admin_state
|
));
|
||||||
.build_admin_system_config_export_payload()
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
BackupScope::Users => {
|
|
||||||
admin_state
|
|
||||||
.build_admin_system_users_export_payload()
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
|
||||||
};
|
};
|
||||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||||
run_backup_with_store(config, &store, payload, Utc::now())
|
warn!(
|
||||||
|
error_category = "object_store_initialization_failed",
|
||||||
|
"S3 backup object store initialization failed"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})?;
|
||||||
|
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
.map_err(|_| {
|
||||||
|
warn!(
|
||||||
|
error_category = "backup_execution_failed",
|
||||||
|
"S3 backup execution failed"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_s3_backup_payload_exclusively(
|
||||||
|
app: &AppState,
|
||||||
|
scope: BackupScope,
|
||||||
|
) -> Result<Value, S3BackupTaskError> {
|
||||||
|
let admin_state = AdminAppState::new(app);
|
||||||
|
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||||
|
match scope {
|
||||||
|
BackupScope::Config => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupScope::Users => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupScope::Data => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
warn!(
|
||||||
|
error_category = "system_import_coordination_failed",
|
||||||
|
lock_error = ?error,
|
||||||
|
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})?
|
||||||
|
.map_err(S3BackupTaskError::from)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||||
|
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||||
|
.ok()
|
||||||
|
.map(|value| value.trim().to_string())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.or_else(|| {
|
||||||
|
app.encryption_key()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_s3_backup_config_for_run(
|
async fn load_s3_backup_config_for_run(
|
||||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
|||||||
.or_else(|| admin_system_config_default_value(key));
|
.or_else(|| admin_system_config_default_value(key));
|
||||||
if let Some(value) = value {
|
if let Some(value) = value {
|
||||||
let value = if *key == "backup_s3_secret_access_key" {
|
let value = if *key == "backup_s3_secret_access_key" {
|
||||||
decrypt_s3_secret_access_key(app, value)?
|
decrypt_s3_secret_access_key(app, value).await?
|
||||||
} else {
|
} else {
|
||||||
value
|
value
|
||||||
};
|
};
|
||||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
|||||||
Ok(values)
|
Ok(values)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
async fn decrypt_s3_secret_access_key(
|
||||||
let Some(ciphertext) = value
|
app: &AppState,
|
||||||
|
value: Value,
|
||||||
|
) -> Result<Value, S3BackupTaskError> {
|
||||||
|
let Some(stored_value) = value
|
||||||
.as_str()
|
.as_str()
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
else {
|
else {
|
||||||
return Ok(value);
|
return Ok(value);
|
||||||
};
|
};
|
||||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||||
else {
|
app,
|
||||||
return Err(S3BackupTaskError::bad_request(
|
"backup_s3_secret_access_key",
|
||||||
|
stored_value.to_string(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
S3BackupTaskError::bad_request(
|
||||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||||
));
|
)
|
||||||
};
|
})?;
|
||||||
Ok(Value::String(plaintext))
|
Ok(Value::String(plaintext))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||||
json!({
|
json!({
|
||||||
"scope": result.scope.as_config_value(),
|
"scope": result.scope.as_config_value(),
|
||||||
"bucket": result.bucket,
|
|
||||||
"object_key": result.object_key,
|
|
||||||
"bytes": result.bytes,
|
"bytes": result.bytes,
|
||||||
"sha256": result.sha256,
|
"sha256": result.sha256,
|
||||||
"export_version": result.export_version,
|
"export_version": result.export_version,
|
||||||
"exported_at": result.exported_at,
|
"exported_at": result.exported_at,
|
||||||
"compression": result.compression,
|
"compression": result.compression,
|
||||||
"deleted_old_objects": result.deleted_old_objects,
|
"encryption": result.encryption,
|
||||||
|
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||||
|
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||||
|
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||||
|
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||||
|
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||||
|
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||||
|
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||||
|
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||||
|
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::convert::Infallible;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||||
@@ -603,9 +801,78 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
|
use crate::handlers::shared::decrypt_system_config_secret;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||||
|
Ok(())
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||||
|
Err(super::BackupLockRenewalFailure::Lost)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||||
|
Err(super::BackupLockRenewalFailure::Backend(
|
||||||
|
"redis unavailable"
|
||||||
|
))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||||
|
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||||
|
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||||
|
let backup = async move {
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||||
|
Ok::<(), super::S3BackupTaskError>(())
|
||||||
|
};
|
||||||
|
let heartbeat = tokio::spawn(async {});
|
||||||
|
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||||
|
assert!(matches!(
|
||||||
|
outcome,
|
||||||
|
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||||
|
));
|
||||||
|
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||||
|
let heartbeat = tokio::spawn(async {});
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
|
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
outcome,
|
||||||
|
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||||
|
let app = AppState::new().expect("app state should build");
|
||||||
|
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||||
|
.await
|
||||||
|
.expect("test should acquire the system import lease");
|
||||||
|
|
||||||
|
let error = super::build_s3_backup_payload_exclusively(
|
||||||
|
&app,
|
||||||
|
crate::backup::scopes::BackupScope::Config,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("backup snapshot must not overlap a system import");
|
||||||
|
|
||||||
|
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||||
|
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||||
|
}
|
||||||
|
|
||||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||||
vec![
|
vec![
|
||||||
(
|
(
|
||||||
@@ -632,6 +899,92 @@ mod tests {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||||
|
let plaintext = "legacy-s3-secret-access-key";
|
||||||
|
let mut entries = valid_s3_backup_config_values();
|
||||||
|
entries
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||||
|
.expect("secret config fixture should exist")
|
||||||
|
.1 = serde_json::json!(plaintext);
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("app state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.with_system_config_values_for_tests(entries),
|
||||||
|
);
|
||||||
|
|
||||||
|
let values = super::load_s3_backup_config_values(&app)
|
||||||
|
.await
|
||||||
|
.expect("legacy S3 config should load");
|
||||||
|
assert_eq!(
|
||||||
|
values.get("backup_s3_secret_access_key"),
|
||||||
|
Some(&serde_json::json!(plaintext))
|
||||||
|
);
|
||||||
|
|
||||||
|
let stored = app
|
||||||
|
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||||
|
.await
|
||||||
|
.expect("stored S3 secret should read")
|
||||||
|
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||||
|
.expect("stored S3 secret should remain a string");
|
||||||
|
assert_ne!(stored, plaintext);
|
||||||
|
assert_eq!(
|
||||||
|
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||||
|
.expect("migrated S3 secret should decrypt"),
|
||||||
|
plaintext
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||||
|
let plaintext = "s3-secret-from-unavailable-key";
|
||||||
|
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||||
|
.expect("unknown-key fixture should encrypt");
|
||||||
|
let mut entries = valid_s3_backup_config_values();
|
||||||
|
entries
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||||
|
.expect("secret config fixture should exist")
|
||||||
|
.1 = serde_json::json!(ciphertext.clone());
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("app state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.with_system_config_values_for_tests(entries),
|
||||||
|
);
|
||||||
|
|
||||||
|
let error = super::load_s3_backup_config_values(&app)
|
||||||
|
.await
|
||||||
|
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||||
|
let error_text = error.to_string();
|
||||||
|
assert!(!error_text.contains(plaintext));
|
||||||
|
assert!(!error_text.contains(&ciphertext));
|
||||||
|
assert_eq!(
|
||||||
|
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||||
|
.await
|
||||||
|
.expect("stored S3 secret should read"),
|
||||||
|
Some(serde_json::json!(ciphertext))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||||
|
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||||
|
"postgresql://admin:[email protected]/aether".to_string(),
|
||||||
|
));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.status(),
|
||||||
|
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
);
|
||||||
|
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||||
|
assert!(!error.detail().contains("database-secret"));
|
||||||
|
}
|
||||||
|
|
||||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||||
let now = now_unix_secs();
|
let now = now_unix_secs();
|
||||||
StoredBackgroundTaskRun {
|
StoredBackgroundTaskRun {
|
||||||
@@ -774,7 +1127,9 @@ mod tests {
|
|||||||
|
|
||||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||||
|
|
||||||
assert!(payload["bucket"].is_string());
|
assert!(payload.get("bucket").is_none());
|
||||||
|
assert!(payload.get("prefix").is_none());
|
||||||
|
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||||
assert!(!payload.to_string().contains("secret"));
|
assert!(!payload.to_string().contains("secret"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
|||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
warn!(
|
||||||
|
error_category = "schedule_tick_failed",
|
||||||
|
"S3 backup schedule tick failed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
|||||||
) -> Result<(), GatewayError> {
|
) -> Result<(), GatewayError> {
|
||||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||||
Ok(values) => values,
|
Ok(values) => values,
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup schedule config load failed");
|
warn!(
|
||||||
|
error_category = "config_load_failed",
|
||||||
|
"S3 backup schedule config load failed"
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let config = match S3BackupConfig::from_json_map(&values) {
|
let config = match S3BackupConfig::from_json_map(&values) {
|
||||||
Ok(config) => config,
|
Ok(config) => config,
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
warn!(
|
||||||
|
error_category = "config_invalid",
|
||||||
|
"S3 backup schedule config is invalid"
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
|||||||
|
|
||||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
warn!(
|
||||||
|
error_category = "task_submission_failed",
|
||||||
|
"S3 backup scheduled task submission failed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
use crate::handlers::shared::{
|
use crate::handlers::shared::{
|
||||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||||
|
system_config_bool, system_config_string,
|
||||||
};
|
};
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
use std::net::{IpAddr, SocketAddr};
|
||||||
|
|
||||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||||
@@ -10,8 +12,19 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
|||||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||||
|
|
||||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||||
|
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||||
|
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||||
|
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||||
|
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||||
|
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||||
|
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||||
|
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||||
|
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||||
|
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||||
|
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||||
|
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct BarkPushConfig {
|
pub(crate) struct BarkPushConfig {
|
||||||
pub(crate) enabled: bool,
|
pub(crate) enabled: bool,
|
||||||
pub(crate) device_key: Option<String>,
|
pub(crate) device_key: Option<String>,
|
||||||
@@ -19,6 +32,21 @@ pub(crate) struct BarkPushConfig {
|
|||||||
pub(crate) template: Option<String>,
|
pub(crate) template: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for BarkPushConfig {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter
|
||||||
|
.debug_struct("BarkPushConfig")
|
||||||
|
.field("enabled", &self.enabled)
|
||||||
|
.field(
|
||||||
|
"device_key",
|
||||||
|
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||||
|
)
|
||||||
|
.field("server_url", &self.server_url)
|
||||||
|
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||||
let value = state
|
let value = state
|
||||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||||
@@ -35,23 +63,34 @@ pub(crate) async fn read_bark_push_config(
|
|||||||
state: &AppState,
|
state: &AppState,
|
||||||
) -> Result<BarkPushConfig, GatewayError> {
|
) -> Result<BarkPushConfig, GatewayError> {
|
||||||
let enabled = bark_push_module_enabled(state).await?;
|
let enabled = bark_push_module_enabled(state).await?;
|
||||||
let device_key = state
|
|
||||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
|
||||||
.await?
|
|
||||||
.and_then(|value| system_config_string(Some(&value)))
|
|
||||||
.map(|value| {
|
|
||||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
|
||||||
});
|
|
||||||
let server_url = state
|
let server_url = state
|
||||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||||
.await?
|
.await?
|
||||||
.and_then(|value| system_config_string(Some(&value)))
|
.and_then(|value| system_config_string(Some(&value)))
|
||||||
.filter(|value| !value.trim().is_empty())
|
.filter(|value| !value.trim().is_empty())
|
||||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||||
|
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||||
|
let server_url = normalized_bark_server_url(&server_url)?;
|
||||||
|
let binding = bark_device_key_binding(&server_url)
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||||
|
let device_key = state
|
||||||
|
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||||
|
.await?
|
||||||
|
.and_then(|value| system_config_string(Some(&value)));
|
||||||
|
let device_key = match device_key {
|
||||||
|
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if let Some(device_key) = device_key.as_deref() {
|
||||||
|
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||||
|
}
|
||||||
let template = state
|
let template = state
|
||||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||||
.await?
|
.await?
|
||||||
.and_then(|value| system_config_string(Some(&value)));
|
.and_then(|value| system_config_string(Some(&value)));
|
||||||
|
if let Some(template) = template.as_deref() {
|
||||||
|
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(BarkPushConfig {
|
Ok(BarkPushConfig {
|
||||||
enabled,
|
enabled,
|
||||||
@@ -62,7 +101,7 @@ pub(crate) async fn read_bark_push_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn send_bark_push(
|
pub(crate) async fn send_bark_push(
|
||||||
state: &AppState,
|
_state: &AppState,
|
||||||
config: &BarkPushConfig,
|
config: &BarkPushConfig,
|
||||||
title: &str,
|
title: &str,
|
||||||
markdown_body: &str,
|
markdown_body: &str,
|
||||||
@@ -76,11 +115,13 @@ pub(crate) async fn send_bark_push(
|
|||||||
"Bark Device Key 不能为空".to_string(),
|
"Bark Device Key 不能为空".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||||
let response = state
|
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||||
.client
|
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||||
.post(format!("{server_url}/push"))
|
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||||
|
let response = client
|
||||||
|
.post(push_url)
|
||||||
.json(&json!({
|
.json(&json!({
|
||||||
"device_key": device_key,
|
"device_key": device_key,
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -88,16 +129,14 @@ pub(crate) async fn send_bark_push(
|
|||||||
}))
|
}))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let text = response
|
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||||
.text()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||||
|
let text = String::from_utf8_lossy(&body);
|
||||||
if !status.is_success() {
|
if !status.is_success() {
|
||||||
return Err(GatewayError::Internal(format!(
|
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||||
"Bark 返回 HTTP {status}: {text}"
|
|
||||||
)));
|
|
||||||
}
|
}
|
||||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||||
let code_is_ok = payload
|
let code_is_ok = payload
|
||||||
@@ -114,53 +153,260 @@ pub(crate) async fn send_bark_push(
|
|||||||
})
|
})
|
||||||
.unwrap_or(true);
|
.unwrap_or(true);
|
||||||
if !code_is_ok {
|
if !code_is_ok {
|
||||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||||
let server_url = server_url.trim().trim_end_matches('/');
|
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||||
if server_url.is_empty() {
|
|
||||||
return Err(GatewayError::Internal(
|
|
||||||
"Bark 服务器地址不能为空".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
|
||||||
return Err(GatewayError::Internal(
|
|
||||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(server_url.to_string())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||||
match template {
|
match error {
|
||||||
Some(template) if !template.trim().is_empty() => template
|
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||||
.replace("{title}", title)
|
format!("Bark 响应超过 {max_bytes} 字节")
|
||||||
.replace("{body}", markdown_body),
|
}
|
||||||
_ => markdown_body.to_string(),
|
aether_http::ResponseBodyReadError::Read(error) => {
|
||||||
|
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||||
|
if error.is_timeout() {
|
||||||
|
"timeout"
|
||||||
|
} else if error.is_connect() {
|
||||||
|
"connect"
|
||||||
|
} else if error.is_request() {
|
||||||
|
"request"
|
||||||
|
} else {
|
||||||
|
"transport"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||||
|
canonical_bark_server_url(server_url)
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_bark_push_client_and_url(
|
||||||
|
server_url: &str,
|
||||||
|
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||||
|
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||||
|
let normalized = normalized_bark_server_url(server_url)?;
|
||||||
|
let mut push_url = url::Url::parse(&normalized)
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||||
|
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||||
|
|
||||||
|
let host = push_url
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||||
|
.to_string();
|
||||||
|
let port = push_url
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||||
|
let addresses = aether_http::lookup_host_with_limits(
|
||||||
|
host.as_str(),
|
||||||
|
port,
|
||||||
|
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
let message = match error.kind() {
|
||||||
|
std::io::ErrorKind::TimedOut => "Bark 服务器 DNS 解析超时",
|
||||||
|
std::io::ErrorKind::InvalidData => "Bark 服务器 DNS 解析返回过多地址",
|
||||||
|
_ => "Bark 服务器 DNS 解析失败",
|
||||||
|
};
|
||||||
|
GatewayError::Internal(message.to_string())
|
||||||
|
})?;
|
||||||
|
let allow_benchmarking_ip = push_url.scheme() == "https"
|
||||||
|
&& push_url.port_or_known_default() == Some(443)
|
||||||
|
&& host.eq_ignore_ascii_case("api.day.app");
|
||||||
|
validate_bark_resolved_addresses(
|
||||||
|
&addresses,
|
||||||
|
env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV),
|
||||||
|
allow_benchmarking_ip,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
push_url
|
||||||
|
.path_segments_mut()
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||||
|
.pop_if_empty()
|
||||||
|
.push("push");
|
||||||
|
|
||||||
|
let mut builder = aether_http::apply_http_client_config(
|
||||||
|
reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.redirect(reqwest::redirect::Policy::none()),
|
||||||
|
&aether_http::HttpClientConfig {
|
||||||
|
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||||
|
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||||
|
http2_adaptive_window: true,
|
||||||
|
..aether_http::HttpClientConfig::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if host.parse::<IpAddr>().is_err() {
|
||||||
|
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||||
|
}
|
||||||
|
let client = builder
|
||||||
|
.build()
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||||
|
Ok((client, push_url))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||||
|
if url.scheme() == "http" && !allow_http {
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_resolved_addresses(
|
||||||
|
addresses: &[SocketAddr],
|
||||||
|
allow_private: bool,
|
||||||
|
allow_benchmarking_ip: bool,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
if addresses.is_empty() {
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !allow_private
|
||||||
|
&& addresses.iter().any(|address| {
|
||||||
|
aether_http::is_private_or_reserved_ip(address.ip())
|
||||||
|
&& !(allow_benchmarking_ip
|
||||||
|
&& aether_http::is_ipv4_benchmarking_fake_ip(address.ip()))
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_flag_enabled(key: &str) -> bool {
|
||||||
|
std::env::var(key).ok().is_some_and(|value| {
|
||||||
|
matches!(
|
||||||
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
|
"1" | "true" | "yes" | "on"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_config_field(
|
||||||
|
field: &str,
|
||||||
|
value: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_content_field(
|
||||||
|
field: &str,
|
||||||
|
value: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
validate_bark_config_field(field, value, max_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render_bark_body(
|
||||||
|
template: Option<&str>,
|
||||||
|
title: &str,
|
||||||
|
markdown_body: &str,
|
||||||
|
) -> Result<String, GatewayError> {
|
||||||
|
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||||
|
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||||
|
let template = template
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
.unwrap_or("{body}");
|
||||||
|
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||||
|
|
||||||
|
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||||
|
let mut cursor = 0usize;
|
||||||
|
while cursor < template.len() {
|
||||||
|
let remaining = &template[cursor..];
|
||||||
|
let title_match = remaining.find("{title}");
|
||||||
|
let body_match = remaining.find("{body}");
|
||||||
|
let next = match (title_match, body_match) {
|
||||||
|
(None, None) => {
|
||||||
|
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||||
|
cursor = template.len();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
(Some(index), None) => (index, "{title}", title),
|
||||||
|
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||||
|
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||||
|
(title_index, "{title}", title)
|
||||||
|
}
|
||||||
|
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||||
|
};
|
||||||
|
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||||
|
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||||
|
cursor += next.0 + next.1.len();
|
||||||
|
}
|
||||||
|
if rendered.is_empty() && template.is_empty() {
|
||||||
|
return Ok(String::new());
|
||||||
|
}
|
||||||
|
Ok(rendered)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||||
|
let next_len = output
|
||||||
|
.len()
|
||||||
|
.checked_add(part.len())
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||||
|
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"Bark rendered body exceeds the allowed size".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
output.push_str(part);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{normalized_bark_server_url, render_bark_body};
|
use super::{
|
||||||
|
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||||
|
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||||
|
};
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bark_body_uses_template_when_provided() {
|
fn bark_body_uses_template_when_provided() {
|
||||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||||
|
.expect("template should render");
|
||||||
assert_eq!(rendered, "告警\n\n原始正文");
|
assert_eq!(rendered, "告警\n\n原始正文");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||||
"原始正文"
|
"原始正文"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||||
|
"原始正文"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||||
|
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||||
|
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||||
|
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||||
|
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -170,4 +416,61 @@ mod tests {
|
|||||||
"https://api.day.app"
|
"https://api.day.app"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||||
|
for invalid in [
|
||||||
|
"https://[email protected]",
|
||||||
|
"https://example.com?target=internal",
|
||||||
|
"https://example.com/#fragment",
|
||||||
|
] {
|
||||||
|
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_http_transport_requires_explicit_opt_in() {
|
||||||
|
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||||
|
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||||
|
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_private_targets_require_explicit_opt_in() {
|
||||||
|
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||||
|
assert!(validate_bark_resolved_addresses(&private, false, false).is_err());
|
||||||
|
assert!(validate_bark_resolved_addresses(&private, true, false).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_builtin_server_allows_benchmarking_ip_only_with_https_default_port() {
|
||||||
|
let fake = [SocketAddr::from(([198, 18, 75, 234], 443))];
|
||||||
|
assert!(validate_bark_resolved_addresses(&fake, false, true).is_ok());
|
||||||
|
assert!(validate_bark_resolved_addresses(
|
||||||
|
&[fake[0], SocketAddr::from(([127, 0, 0, 1], 443))],
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
assert!(validate_bark_resolved_addresses(&fake, false, false).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||||
|
let secret = "bark-secret-query";
|
||||||
|
let error = reqwest::Client::new()
|
||||||
|
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||||
|
|
||||||
|
let message = bark_request_error_message(&error);
|
||||||
|
assert!(!message.contains(secret));
|
||||||
|
assert!(!message.contains("bark.example.test"));
|
||||||
|
|
||||||
|
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||||
|
let message = bark_response_body_error_message(&body_error);
|
||||||
|
assert!(!message.contains(secret));
|
||||||
|
assert!(!message.contains("bark.example.test"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@
|
|||||||
#[path = "support/responses_ws_probe.rs"]
|
#[path = "support/responses_ws_probe.rs"]
|
||||||
mod responses_ws_probe;
|
mod responses_ws_probe;
|
||||||
|
|
||||||
use aether_gateway::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
use aether_gateway::{codex_client_originator, codex_client_user_agent};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use http::header::{AUTHORIZATION, USER_AGENT};
|
use http::header::{AUTHORIZATION, USER_AGENT};
|
||||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||||
@@ -78,14 +78,12 @@ fn handshake_headers(access_token: &str, account_id: &str) -> Result<HeaderMap,
|
|||||||
let mut headers = HeaderMap::new();
|
let mut headers = HeaderMap::new();
|
||||||
headers.insert(AUTHORIZATION, bearer_authorization_value(access_token)?);
|
headers.insert(AUTHORIZATION, bearer_authorization_value(access_token)?);
|
||||||
headers.insert(HeaderName::from_static("chatgpt-account-id"), account_id);
|
headers.insert(HeaderName::from_static("chatgpt-account-id"), account_id);
|
||||||
headers.insert(
|
let user_agent = HeaderValue::from_str(&codex_client_user_agent())
|
||||||
USER_AGENT,
|
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||||
HeaderValue::from_static(CODEX_CLIENT_USER_AGENT),
|
headers.insert(USER_AGENT, user_agent);
|
||||||
);
|
let originator = HeaderValue::from_str(&codex_client_originator())
|
||||||
headers.insert(
|
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||||
HeaderName::from_static("originator"),
|
headers.insert(HeaderName::from_static("originator"), originator);
|
||||||
HeaderValue::from_static(CODEX_CLIENT_ORIGINATOR),
|
|
||||||
);
|
|
||||||
Ok(headers)
|
Ok(headers)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,6 +109,18 @@ mod tests {
|
|||||||
assert!(headers.contains_key("chatgpt-account-id"));
|
assert!(headers.contains_key("chatgpt-account-id"));
|
||||||
assert!(headers.contains_key(USER_AGENT));
|
assert!(headers.contains_key(USER_AGENT));
|
||||||
assert!(headers.contains_key("originator"));
|
assert!(headers.contains_key("originator"));
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get(USER_AGENT)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some(aether_gateway::codex_client_user_agent().as_str())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
headers
|
||||||
|
.get("originator")
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some(aether_gateway::codex_client_originator().as_str())
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
CodexResponsesProbeProfile::sent_header_names(),
|
CodexResponsesProbeProfile::sent_header_names(),
|
||||||
vec![
|
vec![
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
|||||||
|| url.password().is_some()
|
|| url.password().is_some()
|
||||||
|| url.query().is_some()
|
|| url.query().is_some()
|
||||||
|| url.fragment().is_some()
|
|| url.fragment().is_some()
|
||||||
|
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||||
{
|
{
|
||||||
return Err(ProbeFailure::InvalidEndpoint);
|
return Err(ProbeFailure::InvalidEndpoint);
|
||||||
}
|
}
|
||||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
|||||||
|
|
||||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||||
let client = wreq::Client::builder()
|
let client = wreq::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
.connect_timeout(config.turn_timeout)
|
.connect_timeout(config.turn_timeout)
|
||||||
.timeout(config.turn_timeout)
|
.timeout(config.turn_timeout)
|
||||||
.build()
|
.build()
|
||||||
@@ -434,7 +436,14 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn probe_url_rejects_credentials_and_query_strings() {
|
fn probe_url_rejects_credentials_and_query_strings() {
|
||||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||||
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
||||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -422,6 +422,7 @@ mod tests {
|
|||||||
local_rejection: None,
|
local_rejection: None,
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Vendored
+2
@@ -4,6 +4,7 @@ mod auth_runtime;
|
|||||||
mod candidate_page;
|
mod candidate_page;
|
||||||
mod dashboard_response;
|
mod dashboard_response;
|
||||||
mod direct_plan_bypass;
|
mod direct_plan_bypass;
|
||||||
|
mod overview_total;
|
||||||
mod scheduler_affinity;
|
mod scheduler_affinity;
|
||||||
mod system_config;
|
mod system_config;
|
||||||
|
|
||||||
@@ -30,6 +31,7 @@ pub(crate) use candidate_page::{
|
|||||||
};
|
};
|
||||||
pub(crate) use dashboard_response::DashboardResponseCache;
|
pub(crate) use dashboard_response::DashboardResponseCache;
|
||||||
pub(crate) use direct_plan_bypass::DirectPlanBypassCache;
|
pub(crate) use direct_plan_bypass::DirectPlanBypassCache;
|
||||||
|
pub(crate) use overview_total::{OverviewTotalCache, OverviewTotalRead};
|
||||||
pub(crate) use scheduler_affinity::{
|
pub(crate) use scheduler_affinity::{
|
||||||
SchedulerAffinityCache, SchedulerAffinitySnapshotEntry, SchedulerAffinityTarget,
|
SchedulerAffinityCache, SchedulerAffinitySnapshotEntry, SchedulerAffinityTarget,
|
||||||
};
|
};
|
||||||
|
|||||||
+196
@@ -0,0 +1,196 @@
|
|||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use aether_data_contracts::repository::usage::StoredUsageDashboardAnalytics;
|
||||||
|
|
||||||
|
const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||||
|
const FAILURE_BACKOFF: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(crate) struct OverviewTotalCache {
|
||||||
|
state: Mutex<CacheState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct CacheState {
|
||||||
|
value: Option<(Instant, Arc<StoredUsageDashboardAnalytics>)>,
|
||||||
|
refreshing: bool,
|
||||||
|
retry_after: Option<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) enum OverviewTotalRead {
|
||||||
|
Pending,
|
||||||
|
Failed,
|
||||||
|
Ready {
|
||||||
|
snapshot: Arc<StoredUsageDashboardAnalytics>,
|
||||||
|
stale: bool,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Owns the single refresh slot even if the request that launched it disconnects.
|
||||||
|
/// Dropping a cancelled or panicking worker also releases the slot with backoff.
|
||||||
|
pub(crate) struct OverviewTotalRefresh {
|
||||||
|
cache: Arc<OverviewTotalCache>,
|
||||||
|
completed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OverviewTotalCache {
|
||||||
|
pub(crate) fn read(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
now: Instant,
|
||||||
|
) -> (OverviewTotalRead, Option<OverviewTotalRefresh>) {
|
||||||
|
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||||
|
let fresh = state
|
||||||
|
.value
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|(at, _)| now.saturating_duration_since(*at) < FRESH_FOR);
|
||||||
|
let retry_allowed = state.retry_after.is_none_or(|after| now >= after);
|
||||||
|
let refresh = if !fresh && !state.refreshing && retry_allowed {
|
||||||
|
state.refreshing = true;
|
||||||
|
Some(OverviewTotalRefresh {
|
||||||
|
cache: Arc::clone(self),
|
||||||
|
completed: false,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let result = match &state.value {
|
||||||
|
Some((_, snapshot)) => OverviewTotalRead::Ready {
|
||||||
|
snapshot: Arc::clone(snapshot),
|
||||||
|
stale: !fresh,
|
||||||
|
},
|
||||||
|
None if state.refreshing => OverviewTotalRead::Pending,
|
||||||
|
None => OverviewTotalRead::Failed,
|
||||||
|
};
|
||||||
|
(result, refresh)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OverviewTotalRefresh {
|
||||||
|
pub(crate) fn finish(mut self, snapshot: Option<StoredUsageDashboardAnalytics>, now: Instant) {
|
||||||
|
let mut state = self
|
||||||
|
.cache
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
state.refreshing = false;
|
||||||
|
if let Some(snapshot) = snapshot {
|
||||||
|
state.value = Some((now, Arc::new(snapshot)));
|
||||||
|
state.retry_after = None;
|
||||||
|
} else {
|
||||||
|
state.retry_after = Some(now + FAILURE_BACKOFF);
|
||||||
|
}
|
||||||
|
self.completed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for OverviewTotalRefresh {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.completed {
|
||||||
|
let mut state = self
|
||||||
|
.cache
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
|
state.refreshing = false;
|
||||||
|
state.retry_after = Some(Instant::now() + FAILURE_BACKOFF);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn snapshot() -> StoredUsageDashboardAnalytics {
|
||||||
|
let mut snapshot = StoredUsageDashboardAnalytics::default();
|
||||||
|
snapshot.total.generated_at = "2026-09-18T00:00:00Z".into();
|
||||||
|
snapshot.total.read_revision = "revision-1".into();
|
||||||
|
snapshot.total.summary.request_count = 42;
|
||||||
|
snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn concurrent_cold_reads_claim_one_refresh() {
|
||||||
|
let cache = Arc::new(OverviewTotalCache::default());
|
||||||
|
let barrier = Arc::new(std::sync::Barrier::new(16));
|
||||||
|
let now = Instant::now();
|
||||||
|
let workers = (0..16)
|
||||||
|
.map(|_| {
|
||||||
|
let cache = Arc::clone(&cache);
|
||||||
|
let barrier = Arc::clone(&barrier);
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
barrier.wait();
|
||||||
|
let (read, refresh) = cache.read(now);
|
||||||
|
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||||
|
refresh
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let mut refreshes = workers
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|worker| worker.join().unwrap())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(refreshes.len(), 1);
|
||||||
|
refreshes.pop().unwrap().finish(Some(snapshot()), now);
|
||||||
|
let (read, refresh) = cache.read(now);
|
||||||
|
assert!(matches!(
|
||||||
|
read,
|
||||||
|
OverviewTotalRead::Ready { stale: false, .. }
|
||||||
|
));
|
||||||
|
assert!(refresh.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expiration_returns_original_snapshot_and_failed_refresh_preserves_it() {
|
||||||
|
let cache = Arc::new(OverviewTotalCache::default());
|
||||||
|
let now = Instant::now();
|
||||||
|
cache.read(now).1.unwrap().finish(Some(snapshot()), now);
|
||||||
|
assert!(cache
|
||||||
|
.read(now + FRESH_FOR - Duration::from_secs(1))
|
||||||
|
.1
|
||||||
|
.is_none());
|
||||||
|
let expired = now + FRESH_FOR;
|
||||||
|
let (read, refresh) = cache.read(expired);
|
||||||
|
let OverviewTotalRead::Ready {
|
||||||
|
snapshot: old,
|
||||||
|
stale: true,
|
||||||
|
} = read
|
||||||
|
else {
|
||||||
|
panic!("expired success must remain visible")
|
||||||
|
};
|
||||||
|
assert_eq!(old.total.generated_at, "2026-09-18T00:00:00Z");
|
||||||
|
assert_eq!(old.total.read_revision, "revision-1");
|
||||||
|
assert!(cache.read(expired).1.is_none());
|
||||||
|
refresh.unwrap().finish(None, expired);
|
||||||
|
let (read, retry) = cache.read(expired + FAILURE_BACKOFF - Duration::from_secs(1));
|
||||||
|
let OverviewTotalRead::Ready {
|
||||||
|
snapshot: retained,
|
||||||
|
stale: true,
|
||||||
|
} = read
|
||||||
|
else {
|
||||||
|
panic!("failed refresh must retain stale success")
|
||||||
|
};
|
||||||
|
assert!(Arc::ptr_eq(&old, &retained));
|
||||||
|
assert!(retry.is_none());
|
||||||
|
assert!(cache.read(expired + FAILURE_BACKOFF).1.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cold_failure_and_worker_cancellation_back_off_before_retrying() {
|
||||||
|
let cache = Arc::new(OverviewTotalCache::default());
|
||||||
|
let now = Instant::now();
|
||||||
|
cache.read(now).1.unwrap().finish(None, now);
|
||||||
|
let (read, refresh) = cache.read(now + Duration::from_secs(9));
|
||||||
|
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||||
|
assert!(refresh.is_none());
|
||||||
|
let (read, refresh) = cache.read(now + FAILURE_BACKOFF);
|
||||||
|
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||||
|
drop(refresh);
|
||||||
|
let after_cancel = Instant::now();
|
||||||
|
let (read, refresh) = cache.read(after_cancel);
|
||||||
|
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||||
|
assert!(refresh.is_none());
|
||||||
|
assert!(cache.read(after_cancel + FAILURE_BACKOFF).1.is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
|||||||
self.detach_all_loads();
|
self.detach_all_loads();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn invalidate(&self, key: &str) {
|
||||||
|
let Ok(_mutation) = self.mutation.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||||
|
self.entries.remove(&key.to_string());
|
||||||
|
self.detach_all_loads();
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn insert_if_generation(
|
pub(crate) fn insert_if_generation(
|
||||||
&self,
|
&self,
|
||||||
key: String,
|
key: String,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
|||||||
"x-aether-tunnel-affinity-forwarded-by";
|
"x-aether-tunnel-affinity-forwarded-by";
|
||||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||||
"x-aether-tunnel-affinity-owner-instance-id";
|
"x-aether-tunnel-affinity-owner-instance-id";
|
||||||
|
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||||
"local_proxy_passthrough_removed";
|
"local_proxy_passthrough_removed";
|
||||||
@@ -125,6 +126,7 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
|||||||
"/v1/messages/count_tokens",
|
"/v1/messages/count_tokens",
|
||||||
"/v1/responses",
|
"/v1/responses",
|
||||||
"/v1/responses/compact",
|
"/v1/responses/compact",
|
||||||
|
"/v1/memories/trace_summarize",
|
||||||
"/v1/realtime",
|
"/v1/realtime",
|
||||||
"/v1/realtime/calls",
|
"/v1/realtime/calls",
|
||||||
"/v1/live",
|
"/v1/live",
|
||||||
@@ -139,6 +141,8 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
|||||||
"/v1beta/models/{model}/operations/{id}",
|
"/v1beta/models/{model}/operations/{id}",
|
||||||
"/v1beta/operations",
|
"/v1beta/operations",
|
||||||
"/v1beta/operations/{id}",
|
"/v1beta/operations/{id}",
|
||||||
|
"/openai/v1/videos",
|
||||||
|
"/openai/v1/videos/{path...}",
|
||||||
"/v1/videos",
|
"/v1/videos",
|
||||||
"/v1/videos/{path...}",
|
"/v1/videos/{path...}",
|
||||||
"/upload/v1beta/files",
|
"/upload/v1beta/files",
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ pub(crate) fn extract_requested_model(
|
|||||||
body: &Bytes,
|
body: &Bytes,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
if decision.route_family.as_deref() == Some("gemini") {
|
if decision.route_family.as_deref() == Some("gemini") {
|
||||||
if let Some(model) = extract_gemini_model_from_path(uri.path()) {
|
if let Some(model) = extract_gemini_requested_model_from_path(uri.path()) {
|
||||||
return Some(model);
|
return Some(model);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -43,23 +43,46 @@ pub(crate) fn extract_requested_model(
|
|||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn extract_gemini_requested_model_from_path(path: &str) -> Option<String> {
|
||||||
|
let model = extract_gemini_model_from_path(path)?;
|
||||||
|
Some(
|
||||||
|
model
|
||||||
|
.split_once("/operations/")
|
||||||
|
.map(|(model, _)| model)
|
||||||
|
.unwrap_or(model.as_str())
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn extract_request_credentials(
|
pub(super) fn extract_request_credentials(
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: &str,
|
auth_endpoint_signature: &str,
|
||||||
|
) -> GatewayExtractedCredentials {
|
||||||
|
extract_request_credentials_with_trusted_auth(headers, uri, auth_endpoint_signature, cfg!(test))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn extract_request_credentials_with_trusted_auth(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: &str,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> GatewayExtractedCredentials {
|
) -> GatewayExtractedCredentials {
|
||||||
let bundle = GatewayCredentialBundle {
|
let bundle = GatewayCredentialBundle {
|
||||||
authorization_bearer: header_value_str(headers, http::header::AUTHORIZATION.as_str())
|
authorization_bearer: unique_header_value_str(
|
||||||
.as_deref()
|
headers,
|
||||||
.and_then(extract_bearer_token)
|
http::header::AUTHORIZATION.as_str(),
|
||||||
.map(ToOwned::to_owned),
|
)
|
||||||
|
.as_deref()
|
||||||
|
.and_then(extract_bearer_token)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
x_api_key: header_value_str(headers, "x-api-key"),
|
x_api_key: header_value_str(headers, "x-api-key"),
|
||||||
api_key: header_value_str(headers, "api-key"),
|
api_key: header_value_str(headers, "api-key"),
|
||||||
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
||||||
query_key: extract_query_api_key(uri),
|
query_key: extract_query_api_key(uri),
|
||||||
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
||||||
};
|
};
|
||||||
let trusted_headers = extract_trusted_auth_headers(headers);
|
let trusted_headers = extract_trusted_auth_headers(headers, trusted_auth_verified);
|
||||||
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
||||||
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
||||||
|
|
||||||
@@ -71,6 +94,20 @@ pub(super) fn extract_request_credentials(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn unique_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<String> {
|
||||||
|
let mut values = headers.get_all(key).iter();
|
||||||
|
let value = values.next()?;
|
||||||
|
if values.next().is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.ok()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
@@ -85,6 +122,7 @@ pub(in crate::control) fn resolve_gateway_credential_carrier(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
||||||
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -97,13 +135,32 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: &str,
|
auth_endpoint_signature: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: &str,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
let signature = auth_endpoint_signature.trim();
|
let signature = auth_endpoint_signature.trim();
|
||||||
if signature.is_empty() {
|
if signature.is_empty() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
let extracted = extract_request_credentials(headers, uri, signature);
|
let extracted = extract_request_credentials_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
);
|
||||||
let trusted_headers = extracted.trusted_headers;
|
let trusted_headers = extracted.trusted_headers;
|
||||||
let bundle = extracted.bundle;
|
let bundle = extracted.bundle;
|
||||||
if bundle.authorization_bearer.is_none()
|
if bundle.authorization_bearer.is_none()
|
||||||
@@ -135,7 +192,7 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
Some(format!(
|
let raw_cache_identity = format!(
|
||||||
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||||
bundle.authorization_bearer.unwrap_or_default(),
|
bundle.authorization_bearer.unwrap_or_default(),
|
||||||
bundle.x_api_key.unwrap_or_default(),
|
bundle.x_api_key.unwrap_or_default(),
|
||||||
@@ -147,11 +204,26 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
trusted_api_key_id,
|
trusted_api_key_id,
|
||||||
trusted_balance_remaining,
|
trusted_balance_remaining,
|
||||||
trusted_access_allowed,
|
trusted_access_allowed,
|
||||||
))
|
);
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(raw_cache_identity.as_bytes());
|
||||||
|
Some(format!("auth-context:sha256:{:x}", hasher.finalize()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_trusted_auth_headers(headers: &http::HeaderMap) -> Option<GatewayTrustedAuthHeaders> {
|
fn extract_trusted_auth_headers(
|
||||||
if !has_trusted_gateway_marker(headers) {
|
headers: &http::HeaderMap,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
|
) -> Option<GatewayTrustedAuthHeaders> {
|
||||||
|
if !trusted_auth_verified {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
if !header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.starts_with("rust-phase3")
|
||||||
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
||||||
@@ -387,7 +459,7 @@ fn extract_bearer_token(value: &str) -> Option<&str> {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let token = token.trim();
|
let token = token.trim();
|
||||||
if token.is_empty() {
|
if token.is_empty() || token.chars().any(char::is_whitespace) {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(token)
|
Some(token)
|
||||||
@@ -472,6 +544,46 @@ mod tests {
|
|||||||
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_requested_model_handles_gemini_generation_and_operation_paths() {
|
||||||
|
let generation_decision = GatewayControlDecision::synthetic(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:generateContent",
|
||||||
|
Some("ai_public".to_string()),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("generate_content".to_string()),
|
||||||
|
Some("gemini:generate_content".to_string()),
|
||||||
|
);
|
||||||
|
let operation_decision = GatewayControlDecision::synthetic(
|
||||||
|
"/v1beta/models/veo-3/operations/task-123:cancel",
|
||||||
|
Some("ai_public".to_string()),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("video".to_string()),
|
||||||
|
Some("gemini:video".to_string()),
|
||||||
|
);
|
||||||
|
let headers = http::HeaderMap::new();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
extract_requested_model(
|
||||||
|
&generation_decision,
|
||||||
|
&uri("/v1beta/models/gemini-2.5-pro:generateContent"),
|
||||||
|
&headers,
|
||||||
|
&Bytes::new(),
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("gemini-2.5-pro")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
extract_requested_model(
|
||||||
|
&operation_decision,
|
||||||
|
&uri("/v1beta/models/veo-3/operations/task-123:cancel"),
|
||||||
|
&headers,
|
||||||
|
&Bytes::new(),
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("veo-3")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn selects_openai_bearer_as_provider_api_key() {
|
fn selects_openai_bearer_as_provider_api_key() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
@@ -491,6 +603,33 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_duplicate_or_combined_authorization_credentials() {
|
||||||
|
let mut duplicate = http::HeaderMap::new();
|
||||||
|
duplicate.append(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer first-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
duplicate.append(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer second-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
let extracted =
|
||||||
|
extract_request_credentials(&duplicate, &uri("/api/admin/system"), "admin:operational");
|
||||||
|
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||||
|
assert!(extracted.primary.is_none());
|
||||||
|
|
||||||
|
let mut combined = http::HeaderMap::new();
|
||||||
|
combined.insert(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer first-token, Bearer second-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
let extracted =
|
||||||
|
extract_request_credentials(&combined, &uri("/api/admin/system"), "admin:operational");
|
||||||
|
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||||
|
assert!(extracted.primary.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn selects_codex_live_bearer_as_provider_api_key() {
|
fn selects_codex_live_bearer_as_provider_api_key() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
@@ -608,7 +747,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cache_key_includes_cookie_header() {
|
fn cache_key_hashes_cookie_header_instead_of_retaining_session_secret() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
||||||
|
|
||||||
@@ -618,7 +757,8 @@ mod tests {
|
|||||||
"internal:session",
|
"internal:session",
|
||||||
)
|
)
|
||||||
.expect("cache key should exist");
|
.expect("cache key should exist");
|
||||||
assert!(cache_key.contains("session=abc123"));
|
assert!(cache_key.starts_with("auth-context:sha256:"));
|
||||||
|
assert!(!cache_key.contains("session=abc123"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -669,12 +809,10 @@ mod tests {
|
|||||||
.expect("trusted cache key should exist");
|
.expect("trusted cache key should exist");
|
||||||
|
|
||||||
assert_ne!(first, second);
|
assert_ne!(first, second);
|
||||||
assert!(first.contains("user-1"));
|
for raw_identity in ["user-1", "key-1", "1.5", "user-2"] {
|
||||||
assert!(first.contains("key-1"));
|
assert!(!first.contains(raw_identity));
|
||||||
assert!(first.contains("1.5"));
|
assert!(!second.contains(raw_identity));
|
||||||
assert!(first.contains("true"));
|
}
|
||||||
assert!(second.contains("user-2"));
|
|
||||||
assert!(second.contains("false"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use axum::body::Bytes;
|
use axum::body::Bytes;
|
||||||
use axum::http::Uri;
|
use axum::http::Uri;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use super::super::GatewayControlDecision;
|
use super::super::GatewayControlDecision;
|
||||||
use super::credentials::{contains_string, extract_requested_model};
|
use super::credentials::{contains_string, extract_requested_model};
|
||||||
@@ -224,14 +225,38 @@ fn wallet_finite_available_usd(
|
|||||||
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn estimate_execution_plan_cost_upper_bound_usd(
|
pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
plan: &aether_contracts::ExecutionPlan,
|
plan: &aether_contracts::ExecutionPlan,
|
||||||
report_context: Option<&serde_json::Value>,
|
report_context: Option<&serde_json::Value>,
|
||||||
) -> Result<Option<f64>, GatewayError> {
|
) -> Result<Option<f64>, GatewayError> {
|
||||||
let started_at = std::time::Instant::now();
|
let started_at = std::time::Instant::now();
|
||||||
let result =
|
let result = async {
|
||||||
estimate_execution_plan_cost_upper_bound_usd_inner(state, plan, report_context).await;
|
let multiplier_snapshot =
|
||||||
|
aether_data_contracts::repository::usage::billing_multiplier_snapshot(report_context)
|
||||||
|
.map_err(|error| GatewayError::Internal(error.to_string()))?;
|
||||||
|
let estimate = estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||||
|
state,
|
||||||
|
plan,
|
||||||
|
report_context,
|
||||||
|
multiplier_snapshot.is_some(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let Some(snapshot) = multiplier_snapshot else {
|
||||||
|
return Ok(estimate);
|
||||||
|
};
|
||||||
|
// Cache the unmultiplied base estimate so different request snapshots
|
||||||
|
// cannot reuse one another's charge. Pricing validation still runs for
|
||||||
|
// a zero multiplier, even when the request has no finite token bound.
|
||||||
|
if snapshot.multiplier() == 0.0 {
|
||||||
|
return Ok(Some(0.0));
|
||||||
|
}
|
||||||
|
estimate
|
||||||
|
.map(|cost| snapshot.cost(cost))
|
||||||
|
.transpose()
|
||||||
|
.map_err(|error| GatewayError::Internal(error.to_string()))
|
||||||
|
}
|
||||||
|
.await;
|
||||||
observe_gateway_stage_ms(
|
observe_gateway_stage_ms(
|
||||||
"auth_capacity_cost_estimate",
|
"auth_capacity_cost_estimate",
|
||||||
started_at.elapsed().as_millis() as u64,
|
started_at.elapsed().as_millis() as u64,
|
||||||
@@ -243,6 +268,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
|||||||
state: &AppState,
|
state: &AppState,
|
||||||
plan: &aether_contracts::ExecutionPlan,
|
plan: &aether_contracts::ExecutionPlan,
|
||||||
report_context: Option<&serde_json::Value>,
|
report_context: Option<&serde_json::Value>,
|
||||||
|
use_base_cost: bool,
|
||||||
) -> Result<Option<f64>, GatewayError> {
|
) -> Result<Option<f64>, GatewayError> {
|
||||||
let api_format = crate::ai_serving::normalize_api_format_alias(&plan.provider_api_format);
|
let api_format = crate::ai_serving::normalize_api_format_alias(&plan.provider_api_format);
|
||||||
let body_json = plan.body.json_body.as_ref();
|
let body_json = plan.body.json_body.as_ref();
|
||||||
@@ -310,7 +336,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
|||||||
if model_id.is_none() && global_model_name.is_none() {
|
if model_id.is_none() && global_model_name.is_none() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let cache_key = execution_plan_cost_upper_bound_cache_key(
|
let mut cache_key = execution_plan_cost_upper_bound_cache_key(
|
||||||
plan,
|
plan,
|
||||||
model_id,
|
model_id,
|
||||||
global_model_name,
|
global_model_name,
|
||||||
@@ -320,6 +346,11 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
|||||||
requested_processing_tier.as_deref(),
|
requested_processing_tier.as_deref(),
|
||||||
cache_ttl_minutes,
|
cache_ttl_minutes,
|
||||||
);
|
);
|
||||||
|
if use_base_cost {
|
||||||
|
// Legacy requests cache provider Key cost; new requests cache base cost.
|
||||||
|
// These values must never share a cache entry for the same provider Key.
|
||||||
|
cache_key.insert_str(0, "base\x1f");
|
||||||
|
}
|
||||||
let ttl = state.frontdoor_runtime_guards.auth_capacity_cache_ttl;
|
let ttl = state.frontdoor_runtime_guards.auth_capacity_cache_ttl;
|
||||||
if ttl.is_zero() {
|
if ttl.is_zero() {
|
||||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||||
@@ -334,6 +365,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
|||||||
max_output_tokens,
|
max_output_tokens,
|
||||||
requested_processing_tier.as_deref(),
|
requested_processing_tier.as_deref(),
|
||||||
cache_ttl_minutes,
|
cache_ttl_minutes,
|
||||||
|
use_base_cost,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -352,6 +384,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
|||||||
max_output_tokens,
|
max_output_tokens,
|
||||||
requested_processing_tier.as_deref(),
|
requested_processing_tier.as_deref(),
|
||||||
cache_ttl_minutes,
|
cache_ttl_minutes,
|
||||||
|
use_base_cost,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
})
|
})
|
||||||
@@ -370,6 +403,7 @@ async fn calculate_execution_plan_cost_upper_bound(
|
|||||||
max_output_tokens: Option<i64>,
|
max_output_tokens: Option<i64>,
|
||||||
requested_processing_tier: Option<&str>,
|
requested_processing_tier: Option<&str>,
|
||||||
cache_ttl_minutes: Option<i64>,
|
cache_ttl_minutes: Option<i64>,
|
||||||
|
use_base_cost: bool,
|
||||||
) -> Result<Option<f64>, GatewayError> {
|
) -> Result<Option<f64>, GatewayError> {
|
||||||
let context =
|
let context =
|
||||||
load_execution_plan_billing_context(state, plan, model_id, global_model_name).await?;
|
load_execution_plan_billing_context(state, plan, model_id, global_model_name).await?;
|
||||||
@@ -382,11 +416,13 @@ async fn calculate_execution_plan_cost_upper_bound(
|
|||||||
estimate.requested_processing_tier = requested_processing_tier.map(ToOwned::to_owned);
|
estimate.requested_processing_tier = requested_processing_tier.map(ToOwned::to_owned);
|
||||||
estimate.cache_ttl_minutes = cache_ttl_minutes;
|
estimate.cache_ttl_minutes = cache_ttl_minutes;
|
||||||
estimate.max_output_tokens = max_output_tokens;
|
estimate.max_output_tokens = max_output_tokens;
|
||||||
|
let mut pricing = aether_billing::BillingModelPricingSnapshot::from(context);
|
||||||
|
if use_base_cost {
|
||||||
|
pricing.provider_billing_type = None;
|
||||||
|
pricing.provider_api_key_rate_multipliers = None;
|
||||||
|
}
|
||||||
aether_billing::BillingService::new()
|
aether_billing::BillingService::new()
|
||||||
.estimate_authorization_cost_upper_bound(
|
.estimate_authorization_cost_upper_bound(&pricing, &estimate)
|
||||||
&aether_billing::BillingModelPricingSnapshot::from(context),
|
|
||||||
&estimate,
|
|
||||||
)
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -747,6 +783,11 @@ async fn request_model_resolves_to_allowed_model(
|
|||||||
return Ok(false);
|
return Ok(false);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Global model names are a reserved routing namespace, so authorization has to
|
||||||
|
// resolve a request the same way candidate planning will: a provider whose own
|
||||||
|
// model carries the requested name only as an upstream alias must not make the
|
||||||
|
// request resolve to that provider's global model.
|
||||||
|
let mut reserved_global_model_names: BTreeMap<String, Option<String>> = BTreeMap::new();
|
||||||
for api_format in candidate_api_formats_for_model_resolution(&client_api_format) {
|
for api_format in candidate_api_formats_for_model_resolution(&client_api_format) {
|
||||||
let resolution = decision
|
let resolution = decision
|
||||||
.model_directive_policy
|
.model_directive_policy
|
||||||
@@ -762,23 +803,45 @@ async fn request_model_resolves_to_allowed_model(
|
|||||||
.list_minimal_candidate_selection_rows_for_api_format(&api_format)
|
.list_minimal_candidate_selection_rows_for_api_format(&api_format)
|
||||||
.await?
|
.await?
|
||||||
};
|
};
|
||||||
|
let reserved_global_model_name = match reserved_global_model_names.get(routing_model) {
|
||||||
|
Some(cached) => cached.clone(),
|
||||||
|
None => {
|
||||||
|
let reserved_global_model_name =
|
||||||
|
crate::data::candidate_selection::resolve_reserved_global_model_name(
|
||||||
|
state.data.as_ref(),
|
||||||
|
&rows,
|
||||||
|
routing_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
reserved_global_model_names.insert(
|
||||||
|
routing_model.to_string(),
|
||||||
|
reserved_global_model_name.clone(),
|
||||||
|
);
|
||||||
|
reserved_global_model_name
|
||||||
|
}
|
||||||
|
};
|
||||||
let matching_rows = rows
|
let matching_rows = rows
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|row| {
|
.filter(|row| {
|
||||||
aether_scheduler_core::row_supports_requested_model_with_model_directives(
|
aether_scheduler_core::row_supports_requested_model_with_reserved_global_model(
|
||||||
row,
|
row,
|
||||||
routing_model,
|
routing_model,
|
||||||
&api_format,
|
&api_format,
|
||||||
false,
|
false,
|
||||||
|
None,
|
||||||
|
reserved_global_model_name.as_deref(),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let Some(resolved_global_model) =
|
let Some(resolved_global_model) =
|
||||||
aether_scheduler_core::resolve_requested_global_model_name_with_model_directives(
|
aether_scheduler_core::resolve_requested_global_model_name_with_reserved_global_model(
|
||||||
&matching_rows,
|
&matching_rows,
|
||||||
routing_model,
|
routing_model,
|
||||||
&api_format,
|
&api_format,
|
||||||
false,
|
false,
|
||||||
|
None,
|
||||||
|
reserved_global_model_name.as_deref(),
|
||||||
)
|
)
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
@@ -832,10 +895,10 @@ mod tests {
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
available_balance_capacity_usd, execution_plan_balance_capacity_rejection,
|
available_balance_capacity_usd, estimate_execution_plan_cost_upper_bound_usd,
|
||||||
execution_plan_cost_upper_bound_cache_key, max_output_tokens_from_request,
|
execution_plan_balance_capacity_rejection, execution_plan_cost_upper_bound_cache_key,
|
||||||
openai_request_input_is_self_contained, output_choice_count_upper_bound,
|
max_output_tokens_from_request, openai_request_input_is_self_contained,
|
||||||
request_model_local_rejection, GatewayLocalAuthRejection,
|
output_choice_count_upper_bound, request_model_local_rejection, GatewayLocalAuthRejection,
|
||||||
};
|
};
|
||||||
use crate::control::{GatewayControlAuthContext, GatewayControlDecision};
|
use crate::control::{GatewayControlAuthContext, GatewayControlDecision};
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
@@ -925,6 +988,7 @@ mod tests {
|
|||||||
local_rejection: None,
|
local_rejection: None,
|
||||||
allowed_models: Some(allowed_models),
|
allowed_models: Some(allowed_models),
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
});
|
});
|
||||||
decision
|
decision
|
||||||
}
|
}
|
||||||
@@ -2069,6 +2133,166 @@ mod tests {
|
|||||||
assert_eq!(estimate, 6.5);
|
assert_eq!(estimate, 6.5);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn charge_estimate_and_capacity_use_request_multiplier_without_key_cost_or_cache_leaks() {
|
||||||
|
let context = billing_context_with_pricing(
|
||||||
|
Some(json!({"tiers": [{
|
||||||
|
"up_to": null,
|
||||||
|
"input_price_per_1m": 0.0,
|
||||||
|
"output_price_per_1m": 10.0
|
||||||
|
}]})),
|
||||||
|
None,
|
||||||
|
Some(json!({"openai:chat": 2.0})),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let mut state = state_with_quota_and_wallet(quota_availability(15.0, false), context);
|
||||||
|
Arc::make_mut(&mut state.frontdoor_runtime_guards).auth_capacity_cache_ttl =
|
||||||
|
Duration::from_secs(60);
|
||||||
|
let decision = decision_with_allowed_models(vec!["gpt-5".to_string()]);
|
||||||
|
let plan = execution_plan(
|
||||||
|
json!({"model": "gpt-5", "messages": [], "max_tokens": 1_000_000}),
|
||||||
|
"openai:chat",
|
||||||
|
);
|
||||||
|
let legacy = billing_report_context();
|
||||||
|
let mut discounted = legacy.clone();
|
||||||
|
discounted["billing_multiplier_snapshot"] = json!({
|
||||||
|
"version": 1,
|
||||||
|
"factors": {"routing_group": 2.0, "promotion": 0.25},
|
||||||
|
"multiplier": 0.5
|
||||||
|
});
|
||||||
|
let mut marked_up = legacy.clone();
|
||||||
|
marked_up["billing_multiplier_snapshot"] = json!({
|
||||||
|
"version": 1,
|
||||||
|
"factors": {"routing_group": 3.0},
|
||||||
|
"multiplier": 3.0
|
||||||
|
});
|
||||||
|
let mut legacy_group_snapshot = legacy.clone();
|
||||||
|
legacy_group_snapshot["routing_group_billing_multiplier"] = json!(1.0);
|
||||||
|
|
||||||
|
// Reuse the same cache for legacy Key cost, independent request
|
||||||
|
// multipliers, and the old group-only snapshot representation.
|
||||||
|
for (report_context, expected) in [
|
||||||
|
(&legacy, 20.0),
|
||||||
|
(&discounted, 5.0),
|
||||||
|
(&marked_up, 30.0),
|
||||||
|
(&legacy_group_snapshot, 10.0),
|
||||||
|
(&discounted, 5.0),
|
||||||
|
(&legacy, 20.0),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(report_context))
|
||||||
|
.await
|
||||||
|
.expect("charge estimate should resolve"),
|
||||||
|
Some(expected)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
execution_plan_balance_capacity_rejection(&state, &decision, &plan, Some(&discounted))
|
||||||
|
.await
|
||||||
|
.expect("discounted request capacity should resolve"),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
execution_plan_balance_capacity_rejection(&state, &decision, &plan, Some(&marked_up))
|
||||||
|
.await
|
||||||
|
.expect("marked-up request capacity should resolve"),
|
||||||
|
Some(GatewayLocalAuthRejection::BalanceDenied {
|
||||||
|
remaining: Some(15.0)
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn charge_estimate_uses_base_price_when_provider_is_free_tier() {
|
||||||
|
let context = billing_context_with_pricing(
|
||||||
|
Some(json!({"tiers": [{
|
||||||
|
"up_to": null,
|
||||||
|
"input_price_per_1m": 0.0,
|
||||||
|
"output_price_per_1m": 10.0
|
||||||
|
}]})),
|
||||||
|
None,
|
||||||
|
Some(json!({"openai:chat": 0.0})),
|
||||||
|
Some("free_tier"),
|
||||||
|
);
|
||||||
|
let state = state_with_quota_and_wallet(quota_availability(15.0, false), context);
|
||||||
|
let plan = execution_plan(
|
||||||
|
json!({"model": "gpt-5", "messages": [], "max_tokens": 1_000_000}),
|
||||||
|
"openai:chat",
|
||||||
|
);
|
||||||
|
let mut report_context = billing_report_context();
|
||||||
|
assert_eq!(
|
||||||
|
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||||
|
.await
|
||||||
|
.expect("legacy free-tier estimate should resolve"),
|
||||||
|
Some(0.0)
|
||||||
|
);
|
||||||
|
report_context["routing_group_billing_multiplier"] = json!(0.5);
|
||||||
|
assert_eq!(
|
||||||
|
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||||
|
.await
|
||||||
|
.expect("charge estimate should use the model base price"),
|
||||||
|
Some(5.0)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn zero_charge_multiplier_bounds_unknown_cost_but_still_rejects_invalid_pricing() {
|
||||||
|
let context = billing_context_with_pricing(
|
||||||
|
Some(json!({"tiers": [{
|
||||||
|
"up_to": null,
|
||||||
|
"input_price_per_1m": 0.0,
|
||||||
|
"output_price_per_1m": 10.0
|
||||||
|
}]})),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let state = state_with_quota_and_wallet(quota_availability(0.0, false), context);
|
||||||
|
let plan = execution_plan(json!({"model": "gpt-5", "messages": []}), "openai:chat");
|
||||||
|
let mut report_context = billing_report_context();
|
||||||
|
assert_eq!(
|
||||||
|
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||||
|
.await
|
||||||
|
.expect("an unspecified output limit has no finite estimate"),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
report_context["billing_multiplier_snapshot"] = json!({
|
||||||
|
"version": 1,
|
||||||
|
"factors": {"routing_group": 0.0},
|
||||||
|
"multiplier": 0.0
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||||
|
.await
|
||||||
|
.expect("zero multiplier should bound the charge"),
|
||||||
|
Some(0.0)
|
||||||
|
);
|
||||||
|
let invalid_context = billing_context_with_pricing(
|
||||||
|
Some(json!({
|
||||||
|
"tiers": [{"up_to": null, "input_price_per_1m": 1.0}],
|
||||||
|
"processing_tiers": {
|
||||||
|
"priority": {"tiers": [{}], "price_multiplier": -1.0}
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let invalid_state =
|
||||||
|
state_with_quota_and_wallet(quota_availability(0.0, false), invalid_context);
|
||||||
|
let invalid_plan = execution_plan(
|
||||||
|
json!({"model": "gpt-5", "messages": [], "service_tier": "priority"}),
|
||||||
|
"openai:chat",
|
||||||
|
);
|
||||||
|
assert!(estimate_execution_plan_cost_upper_bound_usd(
|
||||||
|
&invalid_state,
|
||||||
|
&invalid_plan,
|
||||||
|
Some(&report_context)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn daily_quota_estimate_treats_free_tier_as_zero_cost() {
|
fn daily_quota_estimate_treats_free_tier_as_zero_cost() {
|
||||||
let context = billing_context_with_pricing(
|
let context = billing_context_with_pricing(
|
||||||
|
|||||||
@@ -7,13 +7,16 @@ mod types;
|
|||||||
pub(crate) use credentials::extract_requested_model;
|
pub(crate) use credentials::extract_requested_model;
|
||||||
pub(super) use credentials::resolve_gateway_credential_carrier;
|
pub(super) use credentials::resolve_gateway_credential_carrier;
|
||||||
pub(crate) use gate::{
|
pub(crate) use gate::{
|
||||||
execution_plan_balance_capacity_rejection, request_model_local_rejection,
|
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
request_model_local_rejection, should_buffer_request_for_local_auth,
|
||||||
|
trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||||
};
|
};
|
||||||
pub(crate) use resolution::{
|
pub(crate) use resolution::{
|
||||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||||
resolve_execution_runtime_auth_context, GatewayAdminPrincipalContext,
|
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||||
GatewayControlAuthContext,
|
GatewayAdminPrincipalContext, GatewayControlAuthContext,
|
||||||
|
};
|
||||||
|
pub(super) use resolution::{
|
||||||
|
resolve_control_decision_auth_with_trusted_auth, ControlDecisionAuthResolution,
|
||||||
};
|
};
|
||||||
pub(super) use resolution::{resolve_control_decision_auth, ControlDecisionAuthResolution};
|
|
||||||
pub(crate) use types::GatewayCredentialCarrier;
|
pub(crate) use types::GatewayCredentialCarrier;
|
||||||
|
|||||||
@@ -4,11 +4,9 @@ use aether_data_contracts::repository::provider_catalog::{
|
|||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
use axum::http::Uri;
|
use axum::http::Uri;
|
||||||
use base64::Engine as _;
|
|
||||||
use hmac::Mac;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::wallet_runtime::{
|
use crate::wallet_runtime::{
|
||||||
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
||||||
@@ -17,7 +15,8 @@ use crate::{AppState, GatewayError};
|
|||||||
|
|
||||||
use super::super::GatewayControlDecision;
|
use super::super::GatewayControlDecision;
|
||||||
use super::credentials::{
|
use super::credentials::{
|
||||||
build_auth_context_cache_key, current_unix_secs, extract_request_credentials,
|
build_auth_context_cache_key, build_auth_context_cache_key_with_trusted_auth,
|
||||||
|
current_unix_secs, extract_request_credentials, extract_request_credentials_with_trusted_auth,
|
||||||
extract_trusted_admin_headers, hash_api_key,
|
extract_trusted_admin_headers, hash_api_key,
|
||||||
};
|
};
|
||||||
use super::gate::GatewayLocalAuthRejection;
|
use super::gate::GatewayLocalAuthRejection;
|
||||||
@@ -27,6 +26,9 @@ use super::types::{
|
|||||||
};
|
};
|
||||||
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
||||||
use crate::headers::header_value_str;
|
use crate::headers::header_value_str;
|
||||||
|
use crate::local_auth_token::{
|
||||||
|
decode_local_auth_token, local_auth_token_identity_matches_user, LocalAuthTokenType,
|
||||||
|
};
|
||||||
|
|
||||||
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
||||||
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
||||||
@@ -93,6 +95,30 @@ pub(crate) struct GatewayControlAuthContext {
|
|||||||
pub(crate) allowed_models: Option<Vec<String>>,
|
pub(crate) allowed_models: Option<Vec<String>>,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub(crate) ip_rules: Option<Vec<String>>,
|
pub(crate) ip_rules: Option<Vec<String>>,
|
||||||
|
/// Credential verifier that established this API-key identity. Long-lived
|
||||||
|
/// executions use it to prove that a later row with the same IDs is still
|
||||||
|
/// the record authenticated by the original request.
|
||||||
|
#[serde(skip)]
|
||||||
|
pub(crate) verified_api_key_hash: Option<VerifiedApiKeyHash>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct VerifiedApiKeyHash(String);
|
||||||
|
|
||||||
|
impl VerifiedApiKeyHash {
|
||||||
|
fn new(value: String) -> Self {
|
||||||
|
Self(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_str(&self) -> &str {
|
||||||
|
self.0.as_str()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for VerifiedApiKeyHash {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str("VerifiedApiKeyHash([REDACTED])")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -109,11 +135,30 @@ pub(in super::super) enum ControlDecisionAuthResolution {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(in super::super) async fn resolve_control_decision_auth(
|
pub(in super::super) async fn resolve_control_decision_auth(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
trace_id: &str,
|
||||||
|
decision: GatewayControlDecision,
|
||||||
|
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||||
|
resolve_control_decision_auth_with_trusted_auth(
|
||||||
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
trace_id,
|
||||||
|
decision,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in super::super) async fn resolve_control_decision_auth_with_trusted_auth(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
mut decision: GatewayControlDecision,
|
mut decision: GatewayControlDecision,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||||
if let Some(admin_principal) =
|
if let Some(admin_principal) =
|
||||||
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
||||||
@@ -132,10 +177,18 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
decision.admin_principal = Some(admin_principal);
|
decision.admin_principal = Some(admin_principal);
|
||||||
}
|
}
|
||||||
|
|
||||||
let auth_context_cache_key = decision
|
let auth_context_cache_key =
|
||||||
.auth_endpoint_signature
|
decision
|
||||||
.as_deref()
|
.auth_endpoint_signature
|
||||||
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature));
|
.as_deref()
|
||||||
|
.and_then(|signature| {
|
||||||
|
build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
let mut resolved_auth_context = None;
|
let mut resolved_auth_context = None;
|
||||||
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
||||||
@@ -149,6 +202,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
decision.auth_endpoint_signature.as_deref(),
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await?,
|
.await?,
|
||||||
);
|
);
|
||||||
@@ -168,6 +222,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
uri,
|
uri,
|
||||||
decision.auth_endpoint_signature.as_deref(),
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
true,
|
true,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -336,7 +391,7 @@ async fn resolve_local_admin_principal(
|
|||||||
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let claims = match decode_local_auth_token(access_token, "access") {
|
let claims = match decode_local_auth_token(access_token, LocalAuthTokenType::Access) {
|
||||||
Ok(claims) => claims,
|
Ok(claims) => claims,
|
||||||
Err(_) => return Ok(None),
|
Err(_) => return Ok(None),
|
||||||
};
|
};
|
||||||
@@ -351,6 +406,14 @@ async fn resolve_local_admin_principal(
|
|||||||
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn resolve_local_admin_session_principal(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
) -> Result<Option<GatewayAdminPrincipalContext>, GatewayError> {
|
||||||
|
resolve_local_admin_principal(state, headers, uri, Some("admin:operational")).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn resolve_local_admin_principal_from_claims(
|
async fn resolve_local_admin_principal_from_claims(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
@@ -373,6 +436,9 @@ async fn resolve_local_admin_principal_from_claims(
|
|||||||
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
if !local_auth_token_identity_matches_user(claims, &user) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
let now = chrono::Utc::now();
|
let now = chrono::Utc::now();
|
||||||
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
||||||
@@ -380,6 +446,7 @@ async fn resolve_local_admin_principal_from_claims(
|
|||||||
};
|
};
|
||||||
if session.is_revoked()
|
if session.is_revoked()
|
||||||
|| session.is_expired(now)
|
|| session.is_expired(now)
|
||||||
|
|| session.security_version != user.security_version
|
||||||
|| session.client_device_id != client_device_id
|
|| session.client_device_id != client_device_id
|
||||||
{
|
{
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -431,68 +498,6 @@ fn local_admin_user_agent(headers: &http::HeaderMap) -> Option<String> {
|
|||||||
.map(|value| value.chars().take(1000).collect())
|
.map(|value| value.chars().take(1000).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn local_auth_secret() -> String {
|
|
||||||
std::env::var("JWT_SECRET_KEY")
|
|
||||||
.ok()
|
|
||||||
.map(|value| value.trim().to_string())
|
|
||||||
.filter(|value| !value.is_empty())
|
|
||||||
.unwrap_or_else(|| "aether-rust-dev-jwt-secret".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode_local_auth_token(
|
|
||||||
token: &str,
|
|
||||||
expected_type: &str,
|
|
||||||
) -> Result<serde_json::Map<String, Value>, String> {
|
|
||||||
let mut parts = token.split('.');
|
|
||||||
let Some(header_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
let Some(payload_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
let Some(signature_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
if parts.next().is_some() {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
let signing_input = format!("{header_segment}.{payload_segment}");
|
|
||||||
let signature = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
|
||||||
.decode(signature_segment)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
let mut mac = hmac::Hmac::<sha2::Sha256>::new_from_slice(local_auth_secret().as_bytes())
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
mac.update(signing_input.as_bytes());
|
|
||||||
mac.verify_slice(&signature)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
|
|
||||||
let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
|
||||||
.decode(payload_segment)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
let payload =
|
|
||||||
serde_json::from_slice::<Value>(&payload_bytes).map_err(|_| "invalid token".to_string())?;
|
|
||||||
let payload = payload
|
|
||||||
.as_object()
|
|
||||||
.cloned()
|
|
||||||
.ok_or_else(|| "invalid token".to_string())?;
|
|
||||||
let actual_type = payload
|
|
||||||
.get("type")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.unwrap_or_default();
|
|
||||||
if actual_type != expected_type {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
}
|
|
||||||
let exp = payload
|
|
||||||
.get("exp")
|
|
||||||
.and_then(Value::as_i64)
|
|
||||||
.ok_or_else(|| "invalid token".to_string())?;
|
|
||||||
if exp <= chrono::Utc::now().timestamp() {
|
|
||||||
return Err("expired token".to_string());
|
|
||||||
}
|
|
||||||
Ok(payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn resolve_execution_runtime_auth_context(
|
pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
decision: &GatewayControlDecision,
|
decision: &GatewayControlDecision,
|
||||||
@@ -525,6 +530,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
|||||||
Some(auth_endpoint_signature),
|
Some(auth_endpoint_signature),
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
|
cfg!(test),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(Some);
|
.map(Some);
|
||||||
@@ -539,6 +545,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
Some(auth_endpoint_signature),
|
Some(auth_endpoint_signature),
|
||||||
true,
|
true,
|
||||||
|
cfg!(test),
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
@@ -558,6 +565,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||||
if is_negative_auth_context(&auth_context)
|
if is_negative_auth_context(&auth_context)
|
||||||
|| !auth_context.access_allowed
|
|| !auth_context.access_allowed
|
||||||
@@ -581,6 +589,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
auth_context.clone(),
|
auth_context.clone(),
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -622,6 +631,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
auth_context,
|
auth_context,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
if refreshed.is_err() {
|
if refreshed.is_err() {
|
||||||
@@ -639,9 +649,16 @@ async fn resolve_security_fresh_auth_context(
|
|||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
stale: GatewayControlAuthContext,
|
stale: GatewayControlAuthContext,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||||
if let Some(refreshed) =
|
if let Some(refreshed) = resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature).await?
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
{
|
{
|
||||||
return Ok(refreshed);
|
return Ok(refreshed);
|
||||||
}
|
}
|
||||||
@@ -660,19 +677,27 @@ async fn resolve_data_backed_auth_context_cached(
|
|||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
cache_negative: bool,
|
cache_negative: bool,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
let Some(cache_key) = cache_key else {
|
let Some(cache_key) = cache_key else {
|
||||||
return resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature)
|
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
.await;
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
};
|
};
|
||||||
loop {
|
loop {
|
||||||
match state.auth_context_cache.register_inflight(cache_key) {
|
match state.auth_context_cache.register_inflight(cache_key) {
|
||||||
AuthContextInflightRegistration::Leader(guard) => {
|
AuthContextInflightRegistration::Leader(guard) => {
|
||||||
let resolved = match resolve_data_backed_auth_context(
|
let resolved = match resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
state,
|
state,
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -708,11 +733,12 @@ async fn resolve_data_backed_auth_context_cached(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
AuthContextInflightRegistration::Bypass => {
|
AuthContextInflightRegistration::Bypass => {
|
||||||
return resolve_data_backed_auth_context(
|
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
state,
|
state,
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -768,28 +794,39 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
|||||||
return Ok((auth_context, None));
|
return Ok((auth_context, None));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let verified_api_key_hash = auth_context.verified_api_key_hash.clone();
|
||||||
let snapshot = {
|
let snapshot = {
|
||||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||||
state
|
if let Some(key_hash) = verified_api_key_hash.as_ref() {
|
||||||
.data
|
state
|
||||||
.read_auth_api_key_snapshot_strong(
|
.data
|
||||||
&auth_context.user_id,
|
.read_auth_api_key_snapshot_by_key_hash_strong(
|
||||||
&auth_context.api_key_id,
|
key_hash.as_str(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
.data
|
||||||
|
.read_auth_api_key_snapshot_strong(
|
||||||
|
&auth_context.user_id,
|
||||||
|
&auth_context.api_key_id,
|
||||||
|
current_unix_secs(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let Some(snapshot) = snapshot else {
|
let Some(snapshot) = snapshot else {
|
||||||
let mut denied = auth_context;
|
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||||
denied.access_allowed = false;
|
};
|
||||||
denied.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
if snapshot.user_id != auth_context.user_id || snapshot.api_key_id != auth_context.api_key_id {
|
||||||
denied.balance_remaining = None;
|
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||||
return Ok((denied, None));
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||||
let refreshed = build_data_backed_auth_context(
|
let mut refreshed = build_data_backed_auth_context(
|
||||||
state,
|
state,
|
||||||
snapshot.clone(),
|
snapshot.clone(),
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
@@ -798,9 +835,19 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
|||||||
wallet_access,
|
wallet_access,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
refreshed.verified_api_key_hash = verified_api_key_hash;
|
||||||
Ok((refreshed, Some(snapshot)))
|
Ok((refreshed, Some(snapshot)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn deny_refreshed_auth_context(
|
||||||
|
mut auth_context: GatewayControlAuthContext,
|
||||||
|
) -> GatewayControlAuthContext {
|
||||||
|
auth_context.access_allowed = false;
|
||||||
|
auth_context.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||||
|
auth_context.balance_remaining = None;
|
||||||
|
auth_context
|
||||||
|
}
|
||||||
|
|
||||||
fn put_cached_auth_context(
|
fn put_cached_auth_context(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
cache_key: String,
|
cache_key: String,
|
||||||
@@ -913,6 +960,23 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
|
resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
let Some(signature) = auth_endpoint_signature
|
let Some(signature) = auth_endpoint_signature
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
@@ -923,7 +987,12 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
if !state.has_auth_api_key_reader() {
|
if !state.has_auth_api_key_reader() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let extracted = extract_request_credentials(headers, uri, signature);
|
let extracted = extract_request_credentials_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
);
|
||||||
let principal = derive_principal_candidate(&extracted);
|
let principal = derive_principal_candidate(&extracted);
|
||||||
let now_unix_secs = current_unix_secs();
|
let now_unix_secs = current_unix_secs();
|
||||||
|
|
||||||
@@ -955,6 +1024,7 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -963,17 +1033,17 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||||
Ok(Some(
|
let mut auth_context = build_data_backed_auth_context(
|
||||||
build_data_backed_auth_context(
|
state,
|
||||||
state,
|
snapshot,
|
||||||
snapshot,
|
signature,
|
||||||
signature,
|
None,
|
||||||
None,
|
None,
|
||||||
None,
|
wallet_access,
|
||||||
wallet_access,
|
)
|
||||||
)
|
.await;
|
||||||
.await,
|
auth_context.verified_api_key_hash = Some(VerifiedApiKeyHash::new(key_hash));
|
||||||
))
|
Ok(Some(auth_context))
|
||||||
}
|
}
|
||||||
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
||||||
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
||||||
@@ -1068,6 +1138,7 @@ async fn resolve_antigravity_bearer_bridge_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1127,6 +1198,7 @@ async fn resolve_trusted_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1158,9 +1230,7 @@ async fn build_data_backed_auth_context(
|
|||||||
let invalid_api_key = !snapshot.user_is_active
|
let invalid_api_key = !snapshot.user_is_active
|
||||||
|| snapshot.user_is_deleted
|
|| snapshot.user_is_deleted
|
||||||
|| !snapshot.api_key_is_active
|
|| !snapshot.api_key_is_active
|
||||||
|| snapshot
|
|| api_key_is_expired(snapshot.api_key_expires_at_unix_secs, current_unix_secs());
|
||||||
.api_key_expires_at_unix_secs
|
|
||||||
.is_some_and(|expires_at| expires_at < current_unix_secs());
|
|
||||||
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
||||||
let key_access_allowed = header_access_allowed
|
let key_access_allowed = header_access_allowed
|
||||||
.map(|value| value && snapshot.currently_usable)
|
.map(|value| value && snapshot.currently_usable)
|
||||||
@@ -1225,9 +1295,14 @@ async fn build_data_backed_auth_context(
|
|||||||
local_rejection,
|
local_rejection,
|
||||||
allowed_models,
|
allowed_models,
|
||||||
ip_rules: snapshot.api_key_ip_rules,
|
ip_rules: snapshot.api_key_ip_rules,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn api_key_is_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
|
||||||
|
expires_at_unix_secs.is_some_and(|expires_at| expires_at <= now_unix_secs)
|
||||||
|
}
|
||||||
|
|
||||||
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
||||||
items.iter().any(|item| api_format_matches(item, target))
|
items.iter().any(|item| api_format_matches(item, target))
|
||||||
}
|
}
|
||||||
@@ -1282,18 +1357,21 @@ async fn auth_snapshot_allows_requested_provider(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if !state.has_provider_catalog_data_reader() {
|
if !state.has_provider_catalog_data_reader() {
|
||||||
return true;
|
debug!(
|
||||||
|
"deny requested provider {}: provider catalog is unavailable for allowlist resolution",
|
||||||
|
requested_provider
|
||||||
|
);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let providers = match state.list_provider_catalog_providers(true).await {
|
let providers = match state.list_provider_catalog_providers(true).await {
|
||||||
Ok(value) => value,
|
Ok(value) => value,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
debug!(
|
warn!(
|
||||||
"skip local provider auth gate for requested provider {}: provider catalog lookup failed: {:?}",
|
"deny requested provider {}: provider catalog lookup failed: {:?}",
|
||||||
requested_provider,
|
requested_provider, err
|
||||||
err
|
|
||||||
);
|
);
|
||||||
return true;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1331,11 +1409,11 @@ async fn auth_snapshot_allows_requested_provider(
|
|||||||
{
|
{
|
||||||
Ok(value) => value,
|
Ok(value) => value,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
debug!(
|
warn!(
|
||||||
"skip local provider auth gate for requested provider {}: provider endpoint lookup failed: {:?}",
|
"deny requested provider {}: provider endpoint lookup failed: {:?}",
|
||||||
requested_provider, err
|
requested_provider, err
|
||||||
);
|
);
|
||||||
return true;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1421,12 +1499,12 @@ fn get_cached_auth_context_with_age(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use aether_data::repository::auth::{
|
use aether_data::repository::auth::{
|
||||||
AuthApiKeyWriteRepository, InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
AuthApiKeyWriteRepository, CreateUserApiKeyRecord, InMemoryAuthApiKeySnapshotRepository,
|
||||||
|
StoredAuthApiKeySnapshot,
|
||||||
};
|
};
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
use aether_data::repository::wallet::{
|
use aether_data::repository::wallet::{
|
||||||
@@ -1441,9 +1519,10 @@ mod tests {
|
|||||||
use futures_util::future::join_all;
|
use futures_util::future::join_all;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
get_cached_auth_context, resolve_control_decision_auth, resolve_data_backed_auth_context,
|
api_key_is_expired, get_cached_auth_context,
|
||||||
resolve_execution_runtime_auth_context, ControlDecisionAuthResolution,
|
refresh_execution_runtime_auth_context_with_snapshot, resolve_control_decision_auth,
|
||||||
GatewayLocalAuthRejection,
|
resolve_data_backed_auth_context, resolve_execution_runtime_auth_context,
|
||||||
|
ControlDecisionAuthResolution, GatewayLocalAuthRejection,
|
||||||
};
|
};
|
||||||
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
||||||
use crate::control::GatewayControlDecision;
|
use crate::control::GatewayControlDecision;
|
||||||
@@ -1481,6 +1560,14 @@ mod tests {
|
|||||||
path.parse().expect("uri should parse")
|
path.parse().expect("uri should parse")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_key_expiry_is_inclusive_at_the_declared_second() {
|
||||||
|
assert!(!api_key_is_expired(None, 100));
|
||||||
|
assert!(!api_key_is_expired(Some(101), 100));
|
||||||
|
assert!(api_key_is_expired(Some(100), 100));
|
||||||
|
assert!(api_key_is_expired(Some(99), 100));
|
||||||
|
}
|
||||||
|
|
||||||
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
||||||
StoredProviderCatalogProvider::new(
|
StoredProviderCatalogProvider::new(
|
||||||
id.to_string(),
|
id.to_string(),
|
||||||
@@ -1507,40 +1594,24 @@ mod tests {
|
|||||||
.expect("endpoint should build")
|
.expect("endpoint should build")
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SqliteAuthConfigNodes {
|
struct PostgresAuthConfigNodes {
|
||||||
first: AppState,
|
first: AppState,
|
||||||
second: AppState,
|
second: AppState,
|
||||||
_database: TemporarySqliteDatabase,
|
_database: aether_testkit::ManagedPostgresServer,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct TemporarySqliteDatabase(PathBuf);
|
async fn postgres_auth_config_nodes(
|
||||||
|
|
||||||
impl Drop for TemporarySqliteDatabase {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
for suffix in ["", "-shm", "-wal"] {
|
|
||||||
let mut path = self.0.as_os_str().to_os_string();
|
|
||||||
path.push(suffix);
|
|
||||||
let _ = std::fs::remove_file(PathBuf::from(path));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sqlite_auth_config_nodes(
|
|
||||||
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
||||||
) -> SqliteAuthConfigNodes {
|
) -> PostgresAuthConfigNodes {
|
||||||
let database_path = std::env::temp_dir().join(format!(
|
let server = aether_testkit::ManagedPostgresServer::start()
|
||||||
"aether-auth-config-{}.sqlite",
|
.await
|
||||||
uuid::Uuid::new_v4()
|
.expect("temporary PostgreSQL should start");
|
||||||
));
|
|
||||||
let mut pool = SqlPoolConfig::default();
|
let mut pool = SqlPoolConfig::default();
|
||||||
pool.min_connections = 0;
|
pool.min_connections = 0;
|
||||||
pool.max_connections = 4;
|
pool.max_connections = 4;
|
||||||
let database = SqlDatabaseConfig::new(
|
let database =
|
||||||
DatabaseDriver::Sqlite,
|
SqlDatabaseConfig::new(DatabaseDriver::Postgres, server.database_url(), pool)
|
||||||
format!("sqlite://{}", database_path.display()),
|
.expect("postgres config should build");
|
||||||
pool,
|
|
||||||
)
|
|
||||||
.expect("sqlite config should build");
|
|
||||||
let config = GatewayDataConfig::from_database_config(database);
|
let config = GatewayDataConfig::from_database_config(database);
|
||||||
let first_data = GatewayDataState::from_config(config.clone())
|
let first_data = GatewayDataState::from_config(config.clone())
|
||||||
.expect("first data state should build")
|
.expect("first data state should build")
|
||||||
@@ -1549,27 +1620,27 @@ mod tests {
|
|||||||
assert!(first_data
|
assert!(first_data
|
||||||
.run_database_migrations()
|
.run_database_migrations()
|
||||||
.await
|
.await
|
||||||
.expect("sqlite migrations should run"));
|
.expect("postgres migrations should run"));
|
||||||
let second_data = GatewayDataState::from_config(config)
|
let second_data = GatewayDataState::from_config(config)
|
||||||
.expect("second data state should build")
|
.expect("second data state should build")
|
||||||
.with_auth_api_key_reader(auth_repository)
|
.with_auth_api_key_reader(auth_repository)
|
||||||
.without_wallet_reader_for_tests();
|
.without_wallet_reader_for_tests();
|
||||||
|
|
||||||
SqliteAuthConfigNodes {
|
PostgresAuthConfigNodes {
|
||||||
first: AppState::new()
|
first: AppState::new()
|
||||||
.expect("first app state should build")
|
.expect("first app state should build")
|
||||||
.with_data_state_for_tests(first_data),
|
.with_data_state_for_tests(first_data),
|
||||||
second: AppState::new()
|
second: AppState::new()
|
||||||
.expect("second app state should build")
|
.expect("second app state should build")
|
||||||
.with_data_state_for_tests(second_data),
|
.with_data_state_for_tests(second_data),
|
||||||
_database: TemporarySqliteDatabase(database_path),
|
_database: server,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
||||||
let nodes =
|
let nodes =
|
||||||
sqlite_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
postgres_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||||
.await;
|
.await;
|
||||||
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
||||||
let old_value = serde_json::json!({"version": "old"});
|
let old_value = serde_json::json!({"version": "old"});
|
||||||
@@ -1769,6 +1840,97 @@ mod tests {
|
|||||||
assert_eq!(repository.touch_count("key-1"), 1);
|
assert_eq!(repository.touch_count("key-1"), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn long_lived_refresh_rejects_same_ids_recreated_with_a_different_credential() {
|
||||||
|
let old_api_key = "sk-old-websocket-credential";
|
||||||
|
let new_api_key = "sk-new-websocket-credential";
|
||||||
|
let old_key_hash = hash_api_key(old_api_key);
|
||||||
|
let new_key_hash = hash_api_key(new_api_key);
|
||||||
|
let mut old_snapshot = sample_snapshot("key-stable-id", "user-stable-id");
|
||||||
|
old_snapshot.user_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||||
|
old_snapshot.api_key_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||||
|
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
|
Some(old_key_hash.clone()),
|
||||||
|
old_snapshot,
|
||||||
|
)]));
|
||||||
|
let data = GatewayDataState::with_auth_api_key_repository_for_tests(repository.clone());
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data);
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
format!("Bearer {old_api_key}").parse().unwrap(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let original = resolve_data_backed_auth_context(
|
||||||
|
&state,
|
||||||
|
&headers,
|
||||||
|
&uri("/v1/responses"),
|
||||||
|
Some("openai:responses"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("initial auth resolution should succeed")
|
||||||
|
.expect("the old API key should authenticate");
|
||||||
|
assert!(original.access_allowed);
|
||||||
|
assert!(original.verified_api_key_hash.is_some());
|
||||||
|
assert!(
|
||||||
|
!format!("{original:?}").contains(&old_key_hash),
|
||||||
|
"the credential verifier must stay redacted from Debug output"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(repository
|
||||||
|
.delete_user_api_key("user-stable-id", "key-stable-id")
|
||||||
|
.await
|
||||||
|
.expect("old API key deletion should succeed"));
|
||||||
|
repository
|
||||||
|
.create_user_api_key(CreateUserApiKeyRecord {
|
||||||
|
user_id: "user-stable-id".to_string(),
|
||||||
|
api_key_id: "key-stable-id".to_string(),
|
||||||
|
key_hash: new_key_hash,
|
||||||
|
key_encrypted: None,
|
||||||
|
name: Some("restored-with-new-secret".to_string()),
|
||||||
|
allowed_providers: Some(vec!["openai".to_string()]),
|
||||||
|
allowed_api_formats: Some(vec!["openai:responses".to_string()]),
|
||||||
|
allowed_models: Some(vec!["gpt-4.1".to_string()]),
|
||||||
|
ip_rules: None,
|
||||||
|
rate_limit: 60,
|
||||||
|
concurrent_limit: Some(5),
|
||||||
|
force_capabilities: None,
|
||||||
|
feature_settings: None,
|
||||||
|
is_active: true,
|
||||||
|
expires_at_unix_secs: Some(4_102_444_800),
|
||||||
|
auto_delete_on_expiry: false,
|
||||||
|
total_requests: 0,
|
||||||
|
total_tokens: 0,
|
||||||
|
total_cost_usd: 0.0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("same-ID API key recreation should resolve")
|
||||||
|
.expect("same-ID API key recreation should persist");
|
||||||
|
|
||||||
|
let (refreshed, snapshot) = refresh_execution_runtime_auth_context_with_snapshot(
|
||||||
|
&state,
|
||||||
|
original,
|
||||||
|
Some("openai:responses"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("long-lived auth refresh should resolve");
|
||||||
|
|
||||||
|
assert!(!refreshed.access_allowed);
|
||||||
|
assert_eq!(
|
||||||
|
refreshed.local_rejection,
|
||||||
|
Some(GatewayLocalAuthRejection::InvalidApiKey)
|
||||||
|
);
|
||||||
|
assert!(snapshot.is_none());
|
||||||
|
assert_eq!(repository.key_hash_lookup_count(&old_key_hash), 1);
|
||||||
|
assert_eq!(
|
||||||
|
repository.snapshot_lookup_count("key-stable-id"),
|
||||||
|
0,
|
||||||
|
"a bound long-lived credential must not fall back to identity-only lookup"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
||||||
let api_key = "sk-test-concurrent-auth-miss";
|
let api_key = "sk-test-concurrent-auth-miss";
|
||||||
@@ -2396,6 +2558,44 @@ mod tests {
|
|||||||
assert_eq!(auth_context.local_rejection, None);
|
assert_eq!(auth_context.local_rejection, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn data_backed_auth_context_denies_unresolved_provider_id_without_catalog_reader() {
|
||||||
|
let api_key = "sk-test-provider-no-catalog";
|
||||||
|
let mut snapshot = sample_snapshot("key-no-catalog", "user-no-catalog");
|
||||||
|
snapshot.user_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||||
|
snapshot.api_key_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||||
|
snapshot.user_allowed_api_formats = None;
|
||||||
|
snapshot.api_key_allowed_api_formats = None;
|
||||||
|
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
|
Some(hash_api_key(api_key)),
|
||||||
|
snapshot,
|
||||||
|
)]));
|
||||||
|
let data = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data);
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("x-api-key", api_key.parse().unwrap());
|
||||||
|
|
||||||
|
let auth_context = resolve_data_backed_auth_context(
|
||||||
|
&state,
|
||||||
|
&headers,
|
||||||
|
&uri("/v1/messages"),
|
||||||
|
Some("claude:messages"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("resolution should succeed")
|
||||||
|
.expect("auth context should exist");
|
||||||
|
|
||||||
|
assert!(!auth_context.access_allowed);
|
||||||
|
assert_eq!(
|
||||||
|
auth_context.local_rejection,
|
||||||
|
Some(GatewayLocalAuthRejection::ProviderNotAllowed {
|
||||||
|
provider: "claude".to_string(),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
||||||
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
||||||
@@ -2410,7 +2610,7 @@ mod tests {
|
|||||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
None, snapshot,
|
None, snapshot,
|
||||||
)]));
|
)]));
|
||||||
let nodes = sqlite_auth_config_nodes(auth_repository.clone()).await;
|
let nodes = postgres_auth_config_nodes(auth_repository.clone()).await;
|
||||||
nodes
|
nodes
|
||||||
.first
|
.first
|
||||||
.auth_context_cache
|
.auth_context_cache
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ pub(super) struct GatewayTrustedAdminHeaders {
|
|||||||
pub(super) management_token_id: Option<String>,
|
pub(super) management_token_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Clone, Default, PartialEq, Eq)]
|
||||||
pub(super) struct GatewayCredentialBundle {
|
pub(super) struct GatewayCredentialBundle {
|
||||||
pub(super) authorization_bearer: Option<String>,
|
pub(super) authorization_bearer: Option<String>,
|
||||||
pub(super) x_api_key: Option<String>,
|
pub(super) x_api_key: Option<String>,
|
||||||
@@ -54,7 +54,25 @@ pub(super) struct GatewayCredentialBundle {
|
|||||||
pub(super) cookie_header: Option<String>,
|
pub(super) cookie_header: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
impl std::fmt::Debug for GatewayCredentialBundle {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let redacted = |value: &Option<String>| value.as_ref().map(|_| "[REDACTED]");
|
||||||
|
formatter
|
||||||
|
.debug_struct("GatewayCredentialBundle")
|
||||||
|
.field(
|
||||||
|
"authorization_bearer",
|
||||||
|
&redacted(&self.authorization_bearer),
|
||||||
|
)
|
||||||
|
.field("x_api_key", &redacted(&self.x_api_key))
|
||||||
|
.field("api_key", &redacted(&self.api_key))
|
||||||
|
.field("x_goog_api_key", &redacted(&self.x_goog_api_key))
|
||||||
|
.field("query_key", &redacted(&self.query_key))
|
||||||
|
.field("cookie_header", &redacted(&self.cookie_header))
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub(super) enum GatewayPrimaryCredential {
|
pub(super) enum GatewayPrimaryCredential {
|
||||||
ProviderApiKey {
|
ProviderApiKey {
|
||||||
raw: String,
|
raw: String,
|
||||||
@@ -70,6 +88,21 @@ pub(super) enum GatewayPrimaryCredential {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for GatewayPrimaryCredential {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let (variant, carrier) = match self {
|
||||||
|
Self::ProviderApiKey { carrier, .. } => ("ProviderApiKey", carrier),
|
||||||
|
Self::BearerToken { carrier, .. } => ("BearerToken", carrier),
|
||||||
|
Self::CookieHeader { carrier, .. } => ("CookieHeader", carrier),
|
||||||
|
};
|
||||||
|
formatter
|
||||||
|
.debug_struct(variant)
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub(super) struct GatewayExtractedCredentials {
|
pub(super) struct GatewayExtractedCredentials {
|
||||||
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
||||||
@@ -78,7 +111,7 @@ pub(super) struct GatewayExtractedCredentials {
|
|||||||
pub(super) primary: Option<GatewayPrimaryCredential>,
|
pub(super) primary: Option<GatewayPrimaryCredential>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Clone, PartialEq)]
|
||||||
pub(super) enum GatewayPrincipalCandidate {
|
pub(super) enum GatewayPrincipalCandidate {
|
||||||
TrustedHeaders(GatewayTrustedAuthHeaders),
|
TrustedHeaders(GatewayTrustedAuthHeaders),
|
||||||
ApiKeyHash {
|
ApiKeyHash {
|
||||||
@@ -94,3 +127,58 @@ pub(super) enum GatewayPrincipalCandidate {
|
|||||||
carrier: GatewayCredentialCarrier,
|
carrier: GatewayCredentialCarrier,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for GatewayPrincipalCandidate {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::TrustedHeaders(headers) => formatter
|
||||||
|
.debug_tuple("TrustedHeaders")
|
||||||
|
.field(headers)
|
||||||
|
.finish(),
|
||||||
|
Self::ApiKeyHash { carrier, .. } => formatter
|
||||||
|
.debug_struct("ApiKeyHash")
|
||||||
|
.field("key_hash", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
Self::DeferredBearerToken { carrier, .. } => formatter
|
||||||
|
.debug_struct("DeferredBearerToken")
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
Self::DeferredCookieHeader { carrier, .. } => formatter
|
||||||
|
.debug_struct("DeferredCookieHeader")
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod debug_redaction_tests {
|
||||||
|
use super::{GatewayCredentialBundle, GatewayCredentialCarrier, GatewayPrimaryCredential};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_credential_debug_output_redacts_raw_authorization_values() {
|
||||||
|
let bundle = GatewayCredentialBundle {
|
||||||
|
authorization_bearer: Some("bundle-bearer-canary".to_string()),
|
||||||
|
api_key: Some("bundle-api-key-canary".to_string()),
|
||||||
|
cookie_header: Some("bundle-cookie-canary".to_string()),
|
||||||
|
..GatewayCredentialBundle::default()
|
||||||
|
};
|
||||||
|
let primary = GatewayPrimaryCredential::ProviderApiKey {
|
||||||
|
raw: "primary-api-key-canary".to_string(),
|
||||||
|
carrier: GatewayCredentialCarrier::ApiKey,
|
||||||
|
};
|
||||||
|
let debug = format!("{bundle:?} {primary:?}");
|
||||||
|
assert!(debug.contains("[REDACTED]"));
|
||||||
|
for secret in [
|
||||||
|
"bundle-bearer-canary",
|
||||||
|
"bundle-api-key-canary",
|
||||||
|
"bundle-cookie-canary",
|
||||||
|
"primary-api-key-canary",
|
||||||
|
] {
|
||||||
|
assert!(!debug.contains(secret), "debug output leaked {secret}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user