mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-05 17:07:46 +08:00
Compare commits
281
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54fbcc25a1 | ||
|
|
d068b62966 | ||
|
|
017d08d45e | ||
|
|
7f45617f79 | ||
|
|
9cc4018a37 | ||
|
|
b49f5c0fd7 | ||
|
|
125cd40aa5 | ||
|
|
491364867e | ||
|
|
8093899b5a | ||
|
|
fa06e9695d | ||
|
|
5d5880d75e | ||
|
|
e8ee7b4ecf | ||
|
|
c1aa5d618d | ||
|
|
00a315e3bf | ||
|
|
f811e29f13 | ||
|
|
fb7e3fc224 | ||
|
|
cabca22d65 | ||
|
|
11989e278e | ||
|
|
532af797bf | ||
|
|
cafa05c4cb | ||
|
|
49ec53cbd9 | ||
|
|
73f1d79637 | ||
|
|
1b8f78a992 | ||
|
|
14ef19d9e1 | ||
|
|
bc0e9f94e2 | ||
|
|
74072e5007 | ||
|
|
c96f8272f4 | ||
|
|
4c07d9fcfb | ||
|
|
45678d9419 | ||
|
|
75bc32cfe9 | ||
|
|
d4bc058c2f | ||
|
|
3541ccfe29 | ||
|
|
bd83cff58f | ||
|
|
85c04335c6 | ||
|
|
c08497c963 | ||
|
|
d30268f80f | ||
|
|
5679375f71 | ||
|
|
3465d23db3 | ||
|
|
e3333f1aef | ||
|
|
31d5e2d172 | ||
|
|
bb9f2eed8e | ||
|
|
926f5cc928 | ||
|
|
27ae884759 | ||
|
|
834eb9c308 | ||
|
|
390b73d4b6 | ||
|
|
bcd121d447 | ||
|
|
3394a51278 | ||
|
|
75471ae4a4 | ||
|
|
57f53903f5 | ||
|
|
81788d3a64 | ||
|
|
30bb0c3130 | ||
|
|
595b8e4e05 | ||
|
|
7e033d0571 | ||
|
|
5745442ed7 | ||
|
|
1a4eba1005 | ||
|
|
2a9d8d3b25 | ||
|
|
e25e240d16 | ||
|
|
cd765f2c2f | ||
|
|
ec95989e02 | ||
|
|
7f5e1a64fe | ||
|
|
f86dd10467 | ||
|
|
69930a6059 | ||
|
|
07cb401fd4 | ||
|
|
593327c803 | ||
|
|
a9a7c64e5d | ||
|
|
70d1a4ab74 | ||
|
|
0b7c7f94ac | ||
|
|
67d0414483 | ||
|
|
e3c01fb554 | ||
|
|
f960bbd2c8 | ||
|
|
906baae88e | ||
|
|
ba7c9f8b27 | ||
|
|
37e3a36680 | ||
|
|
166de33355 | ||
|
|
a95f0d2488 | ||
|
|
0486435f16 | ||
|
|
fb25dde4c9 | ||
|
|
4124749a7d | ||
|
|
5a55116b62 | ||
|
|
681ce56c4f | ||
|
|
bcb2308000 | ||
|
|
6c92db2ba5 | ||
|
|
fdf55525f5 | ||
|
|
4ff4129034 | ||
|
|
364692da55 | ||
|
|
5842c7232e | ||
|
|
72a4bf3408 | ||
|
|
fe1723d87c | ||
|
|
03b198d5ab | ||
|
|
53562fd9de | ||
|
|
e66dd00b84 | ||
|
|
03496c46c5 | ||
|
|
c0ded116a9 | ||
|
|
e5ab73bf35 | ||
|
|
a5456cdc38 | ||
|
|
e9899200f8 | ||
|
|
5a6692ade0 | ||
|
|
6e431e2ff6 | ||
|
|
88df2a2ed5 | ||
|
|
e7864e5611 | ||
|
|
6e6407160a | ||
|
|
01acff0774 | ||
|
|
cc5050155f | ||
|
|
04c4a97766 | ||
|
|
e83399db2f | ||
|
|
f753f14fd2 | ||
|
|
7daf355e65 | ||
|
|
e4f89de90e | ||
|
|
dfe88e34e7 | ||
|
|
c5adcf0312 | ||
|
|
ea24d61910 | ||
|
|
23e0af7b12 | ||
|
|
60b89cc840 | ||
|
|
b748b5bfd2 | ||
|
|
30e36cd09a | ||
|
|
28cd77eb5e | ||
|
|
531f53b443 | ||
|
|
95e4d0149c | ||
|
|
72aea7898b | ||
|
|
8aedf87aad | ||
|
|
e9b64c3e9d | ||
|
|
28f61ec45b | ||
|
|
6aeadcd1d7 | ||
|
|
3a8dadcd6b | ||
|
|
ecc16673eb | ||
|
|
d28dd89039 | ||
|
|
33ea4ebf15 | ||
|
|
8260a87215 | ||
|
|
361952ada9 | ||
|
|
6630856061 | ||
|
|
a893bd0557 | ||
|
|
f2839ae6a7 | ||
|
|
e58570d79d | ||
|
|
99f6499b2b | ||
|
|
17d01d7fe0 | ||
|
|
8b766930b0 | ||
|
|
c7e403b410 | ||
|
|
cf8ea19856 | ||
|
|
7113d04f8a | ||
|
|
099b810a2f | ||
|
|
7aa0c89244 | ||
|
|
7847ae98c6 | ||
|
|
a90d564931 | ||
|
|
a5c3699ae9 | ||
|
|
7b8048c6ae | ||
|
|
ec95f2ca1f | ||
|
|
aa7dbe67d3 | ||
|
|
a26680f460 | ||
|
|
522b979052 | ||
|
|
808946312a | ||
|
|
741107bf71 | ||
|
|
6962731220 | ||
|
|
062e111c03 | ||
|
|
470c59e197 | ||
|
|
2f929e74c7 | ||
|
|
fc0417ceb9 | ||
|
|
44174a31e0 | ||
|
|
b599fb7354 | ||
|
|
14f96c9fa0 | ||
|
|
6948852992 | ||
|
|
1b01b08c31 | ||
|
|
2281f2b754 | ||
|
|
b5ed802277 | ||
|
|
d1b5eb08ee | ||
|
|
dba5e6e9e9 | ||
|
|
c125e78c5f | ||
|
|
d1cb0ebecf | ||
|
|
9d7a0665c0 | ||
|
|
882bb43125 | ||
|
|
db6c522d60 | ||
|
|
e29442a06a | ||
|
|
e15ea0d5d3 | ||
|
|
2f374d6af2 | ||
|
|
4a356f4ea5 | ||
|
|
c7676d567d | ||
|
|
5ca4f87951 | ||
|
|
1fee8954cc | ||
|
|
f69b770f5e | ||
|
|
856accdced | ||
|
|
92749b4d6e | ||
|
|
f08c2e6729 | ||
|
|
e420bc6324 | ||
|
|
d723fb92d3 | ||
|
|
5b1de5f921 | ||
|
|
7ed48e7b58 | ||
|
|
af712ebdbf | ||
|
|
33d5cd5993 | ||
|
|
f5e1420ee6 | ||
|
|
b37b252b14 | ||
|
|
0097ea89ad | ||
|
|
10e63507f0 | ||
|
|
9ff4d73d5c | ||
|
|
0e3bd7eff4 | ||
|
|
1c89b5f9ab | ||
|
|
cdbbda40a6 | ||
|
|
29a9d608d9 | ||
|
|
a6dc43d5f6 | ||
|
|
c6718754d3 | ||
|
|
afdd033745 | ||
|
|
d5f54ffe8b | ||
|
|
f5ec76c5c8 | ||
|
|
784a1e0611 | ||
|
|
507cb33089 | ||
|
|
b08fa3bdb6 | ||
|
|
018af84d7d | ||
|
|
27b0381a9a | ||
|
|
57cdef4b8d | ||
|
|
36e9d21e3f | ||
|
|
b72b6ab137 | ||
|
|
30b2c8548a | ||
|
|
7c5cce4b3c | ||
|
|
9362c34fcd | ||
|
|
344b3031e9 | ||
|
|
e89c3aa674 | ||
|
|
ddbbf835af | ||
|
|
cb58a63ee3 | ||
|
|
6b1074cfcd | ||
|
|
d6894b5532 | ||
|
|
635c6765d9 | ||
|
|
86f7cc0d58 | ||
|
|
206995645b | ||
|
|
9282cce1d6 | ||
|
|
c005700a7e | ||
|
|
14744abd57 | ||
|
|
66d6c17d2d | ||
|
|
c142d39951 | ||
|
|
1eb2d10dec | ||
|
|
dabaeb8dfa | ||
|
|
2d17d4b73f | ||
|
|
18d78dd6c9 | ||
|
|
499942e3e7 | ||
|
|
ba11a72214 | ||
|
|
12571764bc | ||
|
|
1e13fa032c | ||
|
|
47b21a25d3 | ||
|
|
45a3ba8829 | ||
|
|
03f2914044 | ||
|
|
c8d1ae3e7e | ||
|
|
c5ae9c2c77 | ||
|
|
4e47c00154 | ||
|
|
313a637982 | ||
|
|
fe8ff268df | ||
|
|
bac6d6866a | ||
|
|
579f2c7cc1 | ||
|
|
ddcbeb3ae9 | ||
|
|
e25fc984af | ||
|
|
4cf47b1dee | ||
|
|
95cbd43097 | ||
|
|
09005939bf | ||
|
|
7b612b8b5a | ||
|
|
670d5e8d33 | ||
|
|
1de2e70d41 | ||
|
|
89b57464d2 | ||
|
|
09ef3adf70 | ||
|
|
3dfc15963c | ||
|
|
f6884eb8c4 | ||
|
|
f8b4382a54 | ||
|
|
d78b5a81fb | ||
|
|
89fe9e9f0a | ||
|
|
4291a91dc0 | ||
|
|
4c6bafe255 | ||
|
|
979dbc4b33 | ||
|
|
9309ad844f | ||
|
|
d672ba2068 | ||
|
|
587486ab0c | ||
|
|
40a5e1470d | ||
|
|
058660ec2e | ||
|
|
668bf5e40f | ||
|
|
77f93c638d | ||
|
|
d0c0996b9f | ||
|
|
76fb8905c9 | ||
|
|
f822df6cce | ||
|
|
45c840b8d3 | ||
|
|
214f3d6406 | ||
|
|
cc6f5e89b6 | ||
|
|
2ed2cc66ef | ||
|
|
b1bf7837cf | ||
|
|
77229943d1 | ||
|
|
a0369cf49a | ||
|
|
dbbe7b22ab | ||
|
|
d5f34b2ee2 |
@@ -0,0 +1,6 @@
|
||||
# 不固定 test-threads:nextest 默认按 num-cpus 并发,固定值会在更大规格的
|
||||
# runner 或本地开发机上主动压低并发、反而变慢,且无法表达 min(4, num-cpus)。
|
||||
# 这里只保留卡死保护,避免单个挂起用例拖满整个 job。
|
||||
[profile.default]
|
||||
# 60 秒后标记慢测试,连续两轮仍未结束则终止;超时结果保持失败,不隐藏回归。
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
+96
-8
@@ -26,21 +26,25 @@ RUST_LOG=aether_gateway=info
|
||||
# 示例: http://localhost:5173,https://app.example.com
|
||||
# CORS_ORIGINS=http://localhost:5173
|
||||
# CORS_ALLOW_CREDENTIALS=true
|
||||
# 如果前后端跨站并依赖登录刷新 Cookie,还要配合:
|
||||
# 登录刷新 Cookie 对同源浏览器请求和可信反代自动适配 HTTP/HTTPS。
|
||||
# HTTP 自动使用兼容的 SameSite=Lax(显式 Strict 保留);HTTPS 保留原有 SameSite 配置。
|
||||
# 无法确认访问协议时保留安全默认值;HTTPS 反代请正确传递 X-Forwarded-Proto。
|
||||
# AUTH_REFRESH_COOKIE_SECURE 可显式覆盖自动判断,公网部署仍建议使用 HTTPS。
|
||||
# 如果前后端跨站并依赖登录刷新 Cookie,必须使用 HTTPS,并配合:
|
||||
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
||||
# AUTH_REFRESH_COOKIE_SECURE=true
|
||||
|
||||
# 数据库配置
|
||||
# 数据库配置(仅支持 PostgreSQL)
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_NAME=aether
|
||||
DB_PASSWORD=aether
|
||||
DB_PASSWORD=
|
||||
|
||||
# Redis 配置
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=aether
|
||||
REDIS_PASSWORD=
|
||||
|
||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||
@@ -50,6 +54,10 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||
|
||||
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||
|
||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||
ADMIN_EMAIL=[email protected]
|
||||
@@ -75,15 +83,95 @@ ADMIN_USERNAME=admin123456
|
||||
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||
# 普通 PostgreSQL 连接默认语句超时 30 秒、锁等待超时 3 秒;0 显式关闭。
|
||||
# 这是单条 SQL 的期限,不是整个事务总期限;迁移和历史 backfill 使用独立连接放宽。
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS=30000
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS=3000
|
||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||
# 所有监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和升级后的 WebSocket。
|
||||
# 未设置或 0 时按请求上限 + WebSocket 上限推导,最大 65536;显式配置也受 FD 余量限制。
|
||||
# 已知 FD soft limit 时最多 max(1, (FD - 256) / 2),不是整个进程的 FD/内存保证。
|
||||
# 满额的新连接在 HTTP 解析前关闭,不排队创建任务,也不会返回 HTTP 429/503。
|
||||
# AETHER_GATEWAY_MAX_HTTP_CONNECTIONS=4096
|
||||
# 停机先等待 HTTP 请求,再排空本地用量写入;以下期限单位为毫秒。
|
||||
# 进程管理器的强杀期限应覆盖两阶段之和,再预留至少 10 秒收尾。
|
||||
# AETHER_GATEWAY_HTTP_SHUTDOWN_TIMEOUT_MS=30000
|
||||
# AETHER_GATEWAY_USAGE_SHUTDOWN_TIMEOUT_MS=30000
|
||||
# 每客户端、每 origin 的上游空闲连接缓存;不限制活动请求或流持续时间。
|
||||
# AETHER_GATEWAY_UPSTREAM_POOL_MAX_IDLE_PER_HOST=32
|
||||
# AETHER_GATEWAY_UPSTREAM_POOL_IDLE_TIMEOUT_MS=15000
|
||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||
# 请求体按实际缓冲增长申请额度,解压同时计入输入和输出;额度不足返回 503。
|
||||
# 请求体完整读取总超时默认 120000 毫秒;非零值限制在 1000-600000,显式 0 关闭。
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
||||
# 上游流首包后空闲超时默认 300000 毫秒;执行配置 read_ms 优先,显式 0 关闭。
|
||||
# AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS=300000
|
||||
# 流式响应诊断捕获共享预算默认 128 MiB,包含 provider/client 分配;不足时截断审计副本。
|
||||
# 显式 0 关闭此类捕获;不限制协议解析、终态编码和 usage 队列的总内存。
|
||||
# AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||
# usage 诊断正文共享预算,按 JSON 堆内存估算,默认 128 MiB。
|
||||
# 覆盖终态队列 seed、Redis 解码事件、数据库写入 DTO 及正文副本;额度随正文释放。
|
||||
# 不足或显式 0 时先保留计费事实再舍弃正文;已有清空/禁用状态保留,其余标为截断。
|
||||
# 不包含原始 Redis 批次、解码临时分配、序列化和压缩结果、协议观察缓冲或进程总内存。
|
||||
# AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||
# 新 usage 队列消息的完整 JSON payload 上限,默认 1 MiB;显式 0 非法。
|
||||
# 超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义则拒绝入队,终态尝试受限落库,失败即明确失败。
|
||||
# 不限制存量 Redis 消息、整个读取批次、DLQ 或进程总内存。
|
||||
# usage_runtime_queue_payload_* 降级/拒绝计数包含入队和重试预校验的编码尝试,不代表唯一事件数。
|
||||
# AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES=1048576
|
||||
# usage worker 读取/重领共用的逻辑 payload 预留:全进程默认 128 MiB,单批目标 8 MiB。
|
||||
# 按当前消息上限推导 COUNT,默认最多 8 条;预留覆盖整批处理和确认,额度不足等待。
|
||||
# 当前消息上限不能超过总预留额度;单批目标不足一条时仍读一条。0 或非法值回退默认。
|
||||
# 历史/其他生产者的大消息继续处理并计数,不是 RESP、实际堆内存或 DLQ 的硬上限。
|
||||
# AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES=134217728
|
||||
# AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES=8388608
|
||||
# DLQ 原文和最坏 JSON 编码独立预留默认 64 MiB,后台编码及写入最多同时 4 个任务。
|
||||
# 入场前一次预留,额度占满或单条超预算立即失败并保留 pending 原消息,后续重领。
|
||||
# 编码失败不阻塞同批其余正常消息;批次仍报告失败,只有成功项会被确认。
|
||||
# JSON 按字符串最多 6 倍转义保守估算;不截原账务字段,不包含 Redis 命令/连接副本或 RSS。
|
||||
# 0/非法值回退默认;bytes 最大约 4 GiB,jobs 最大 128。超大存量可能需调高额度后恢复。
|
||||
# AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES=67108864
|
||||
# AETHER_USAGE_DLQ_ENCODING_MAX_JOBS=4
|
||||
# 内置 Redis 死信转移要求 Redis 7+ 及 EVAL/TYPE/XPENDING/XADD/XACK/XDEL 权限。
|
||||
# stream 与 DLQ 不能同名;Cluster 还要求两键同 slot,现有默认键未自动迁移。
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式 0 仍受 256 MiB 硬上限保护。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
||||
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
|
||||
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
|
||||
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
|
||||
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||
# 跨网关 relay 解析到受控私有地址时才显式开启;默认关闭以防止被篡改的 attachment
|
||||
# 记录诱导网关向内网转发 relay 凭据。该开关不放宽普通 provider 的目标地址策略。
|
||||
# AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS=false
|
||||
# 更推荐按 relay 主机名精确放行私网部署(逗号分隔,大小写不敏感);不支持通配符/后缀。
|
||||
# AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST=gateway-a.internal,gateway-b.internal
|
||||
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||
# AETHER_BARK_ALLOW_HTTP=false
|
||||
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||
|
||||
# 普通 Provider 反代(包括 Provider OAuth)不按 DNS 地址过滤上游,兼容任意
|
||||
# Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。
|
||||
# URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。
|
||||
|
||||
# 可选 Provider OAuth 客户端。Gemini CLI 和 Antigravity 默认使用内置 native-app
|
||||
# 客户端凭据;自定义 client ID 时必须同时配置对应的 client secret。
|
||||
# 显式配置的 client secret 优先于默认值。
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||
|
||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||
|
||||
@@ -6,7 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: build-tunnel-${{ github.ref }}
|
||||
@@ -17,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Ensure tunnel tag matches Cargo version
|
||||
shell: bash
|
||||
@@ -78,10 +79,10 @@ jobs:
|
||||
use_cross: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
@@ -89,14 +90,14 @@ jobs:
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
workspaces: apps/aether-tunnel -> target
|
||||
key: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
working-directory: apps/aether-tunnel
|
||||
@@ -122,9 +123,10 @@ jobs:
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-tunnel-${{ matrix.name }}
|
||||
path: |
|
||||
@@ -137,9 +139,14 @@ jobs:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: artifacts
|
||||
@@ -148,6 +155,20 @@ jobs:
|
||||
working-directory: artifacts
|
||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||
|
||||
- name: Attest tunnel release provenance
|
||||
id: attest-release
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
artifacts/aether-tunnel-*.tar.gz
|
||||
artifacts/aether-tunnel-*.zip
|
||||
artifacts/SHA256SUMS.txt
|
||||
|
||||
- name: Bundle tunnel release provenance
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Delete stale draft releases for tag
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -170,12 +191,13 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
name: "${{ github.ref_name }}"
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
artifacts/aether-tunnel-*
|
||||
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
artifacts/SHA256SUMS.txt
|
||||
fail_on_unmatched_files: true
|
||||
|
||||
@@ -183,8 +205,10 @@ jobs:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: main
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
@@ -46,10 +44,10 @@ jobs:
|
||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
@@ -77,10 +75,10 @@ jobs:
|
||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: frontend/dist
|
||||
|
||||
@@ -90,7 +88,10 @@ jobs:
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
id-token: write
|
||||
pages: write
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
@@ -25,7 +25,6 @@ env:
|
||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: '1'
|
||||
GHCR_IMAGE: ghcr.io/fawney19/aether
|
||||
|
||||
jobs:
|
||||
source:
|
||||
@@ -36,6 +35,7 @@ jobs:
|
||||
sha: ${{ steps.snapshot.outputs.sha }}
|
||||
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
||||
date: ${{ steps.snapshot.outputs.date }}
|
||||
ghcr_image: ${{ steps.snapshot.outputs.ghcr_image }}
|
||||
steps:
|
||||
- name: Require main branch
|
||||
id: snapshot
|
||||
@@ -49,9 +49,13 @@ jobs:
|
||||
fi
|
||||
|
||||
sha="${GITHUB_SHA}"
|
||||
# Docker 镜像仓库名必须全小写;GitHub owner 可能保留大写,先统一规范化。
|
||||
repository_owner="${GITHUB_REPOSITORY%%/*}"
|
||||
repository_owner="${repository_owner,,}"
|
||||
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
||||
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
||||
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
||||
echo "ghcr_image=ghcr.io/${repository_owner}/aether" >> "${GITHUB_OUTPUT}"
|
||||
echo "Building main at ${sha}."
|
||||
|
||||
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
||||
@@ -59,6 +63,8 @@ jobs:
|
||||
name: Rust CI
|
||||
needs: source
|
||||
uses: ./.github/workflows/rust-ci.yml
|
||||
with:
|
||||
full_scope: true
|
||||
|
||||
rust_extended:
|
||||
name: Rust extended checks
|
||||
@@ -66,12 +72,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
@@ -79,13 +85,13 @@ jobs:
|
||||
run: rustc -Vv
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check all workspace targets
|
||||
env:
|
||||
@@ -112,12 +118,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -157,7 +163,7 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -171,12 +177,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -247,38 +253,26 @@ jobs:
|
||||
arch: arm64
|
||||
os: ubuntu-latest
|
||||
use_cross: true
|
||||
- name: macos-amd64
|
||||
target: x86_64-apple-darwin
|
||||
platform: macos
|
||||
arch: amd64
|
||||
os: macos-15-intel
|
||||
use_cross: false
|
||||
- name: macos-arm64
|
||||
target: aarch64-apple-darwin
|
||||
platform: macos
|
||||
arch: arm64
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build release binary
|
||||
env:
|
||||
@@ -295,7 +289,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -308,17 +302,19 @@ jobs:
|
||||
needs: [source, checks, build]
|
||||
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download Linux binaries and frontend
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -335,20 +331,20 @@ jobs:
|
||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push nightly image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -372,12 +368,12 @@ jobs:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download nightly artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -392,31 +388,29 @@ jobs:
|
||||
VERSION="nightly"
|
||||
|
||||
mkdir -p package release-assets
|
||||
for platform in linux macos; do
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p "${root}/bin" "${root}/frontend"
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-linux-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p "${root}/bin" "${root}/frontend"
|
||||
|
||||
install -m 0755 \
|
||||
"artifacts/nightly-gateway-${platform}-${arch}/aether-gateway" \
|
||||
"${root}/bin/aether-gateway"
|
||||
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
install -m 0755 \
|
||||
"artifacts/nightly-gateway-linux-${arch}/aether-gateway" \
|
||||
"${root}/bin/aether-gateway"
|
||||
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
|
||||
sed \
|
||||
@@ -426,15 +420,15 @@ jobs:
|
||||
chmod 0755 release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 4
|
||||
test "$(wc -l < release-assets/SHA256SUMS)" -eq 4
|
||||
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 2
|
||||
test "$(wc -l < release-assets/SHA256SUMS)" -eq 2
|
||||
(cd release-assets && sha256sum -c SHA256SUMS)
|
||||
for archive in release-assets/*.tar.gz; do
|
||||
tar -tzf "${archive}" >/dev/null
|
||||
done
|
||||
|
||||
- name: Upload nightly package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets/*
|
||||
@@ -452,7 +446,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download nightly package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets
|
||||
@@ -466,6 +460,7 @@ jobs:
|
||||
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
||||
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
||||
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -510,6 +505,15 @@ jobs:
|
||||
--repo "${REPOSITORY}" \
|
||||
--clobber
|
||||
|
||||
published_assets="$(gh release view "${RELEASE_TAG}" --repo "${REPOSITORY}" --json assets --jq '.assets[].name')"
|
||||
while IFS= read -r asset_name; do
|
||||
if [[ "${asset_name}" == aether-nightly-*.tar.gz && ! -f "release-assets/${asset_name}" ]]; then
|
||||
gh release delete-asset "${RELEASE_TAG}" "${asset_name}" \
|
||||
--repo "${REPOSITORY}" \
|
||||
--yes
|
||||
fi
|
||||
done <<<"${published_assets}"
|
||||
|
||||
# target_commitish does not move an existing git tag. Move the ref
|
||||
# only after the complete asset set is available.
|
||||
if gh api "repos/${REPOSITORY}/git/ref/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
|
||||
@@ -534,8 +538,6 @@ jobs:
|
||||
expected_assets=(
|
||||
aether-nightly-linux-amd64.tar.gz
|
||||
aether-nightly-linux-arm64.tar.gz
|
||||
aether-nightly-macos-amd64.tar.gz
|
||||
aether-nightly-macos-arm64.tar.gz
|
||||
SHA256SUMS
|
||||
install.sh
|
||||
)
|
||||
|
||||
@@ -6,8 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-aether-${{ github.ref }}
|
||||
@@ -70,10 +70,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -106,10 +106,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -159,7 +159,7 @@ jobs:
|
||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||
|
||||
- name: Upload VSIX artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||
@@ -186,35 +186,23 @@ jobs:
|
||||
arch: arm64
|
||||
os: ubuntu-latest
|
||||
use_cross: true
|
||||
- name: macos-amd64
|
||||
target: x86_64-apple-darwin
|
||||
platform: macos
|
||||
arch: amd64
|
||||
os: macos-15-intel
|
||||
use_cross: false
|
||||
- name: macos-arm64
|
||||
target: aarch64-apple-darwin
|
||||
platform: macos
|
||||
arch: arm64
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
@@ -230,7 +218,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -242,11 +230,17 @@ jobs:
|
||||
needs: [preflight, frontend, build]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -259,27 +253,27 @@ jobs:
|
||||
cp -r artifacts/frontend-dist dist/frontend
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
@@ -295,7 +289,8 @@ jobs:
|
||||
latest=false
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
id: push
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -304,15 +299,36 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Attest GHCR image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
- name: Attest Docker Hub image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
package:
|
||||
name: Release tarballs
|
||||
needs: [preflight, frontend, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -328,31 +344,29 @@ jobs:
|
||||
fi
|
||||
|
||||
mkdir -p package release-assets
|
||||
for platform in linux macos; do
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p \
|
||||
"${root}/bin" \
|
||||
"${root}/frontend"
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-linux-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p \
|
||||
"${root}/bin" \
|
||||
"${root}/frontend"
|
||||
|
||||
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
install -m 0755 "artifacts/aether-gateway-linux-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
|
||||
sed \
|
||||
@@ -362,8 +376,24 @@ jobs:
|
||||
chmod +x release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
- name: Attest release package provenance
|
||||
id: attest-release
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/install.sh
|
||||
release-assets/SHA256SUMS
|
||||
|
||||
- name: Bundle release package provenance
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Upload release package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets/*
|
||||
@@ -375,15 +405,18 @@ jobs:
|
||||
needs: [preflight, docker, package, vscodex]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download release package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
|
||||
- name: Download VSIX artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: release-assets
|
||||
@@ -410,13 +443,14 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Publish GitHub Release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
generate_release_notes: true
|
||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||
files: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
release-assets/SHA256SUMS
|
||||
release-assets/install.sh
|
||||
release-assets/*.vsix
|
||||
|
||||
+315
-221
@@ -2,6 +2,12 @@ name: Rust CI
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
full_scope:
|
||||
description: "Run all Rust and shell scopes, used by Nightly"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -9,18 +15,58 @@ on:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
|
||||
concurrency:
|
||||
group: rust-ci-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
@@ -36,14 +82,106 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
fmt:
|
||||
name: Format
|
||||
changes:
|
||||
name: Detect Rust CI scope
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
rust: ${{ steps.scope.outputs.rust }}
|
||||
shell: ${{ steps.scope.outputs.shell }}
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Classify changed paths
|
||||
id: scope
|
||||
shell: bash
|
||||
env:
|
||||
RUST_CI_FULL_SCOPE: ${{ inputs.full_scope || false }}
|
||||
run: |
|
||||
# 任何命令失败都必须让本 job 失败,否则 git fetch/diff 出错后仍会写出
|
||||
# rust=false/shell=false,下游会误判为“无需测试”而假绿放行。
|
||||
set -euo pipefail
|
||||
|
||||
# Nightly 通过 workflow_call 显式传入 full_scope;普通 push/PR 只按源码和构建
|
||||
# 指纹触发 Rust jobs,安装脚本、Compose、README 等由 shell scope 覆盖。
|
||||
if [ "$RUST_CI_FULL_SCOPE" = "true" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] \
|
||||
&& [ -n "${GITHUB_BASE_REF:-}" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
git fetch --no-tags origin "$GITHUB_BASE_REF" --depth=1
|
||||
changed_paths=$(git diff --name-only "origin/$GITHUB_BASE_REF...$GITHUB_SHA")
|
||||
elif [ "$GITHUB_EVENT_NAME" = "push" ] \
|
||||
&& [ -n "${GITHUB_EVENT_BEFORE:-}" ] \
|
||||
&& [ "$GITHUB_EVENT_BEFORE" != "0000000000000000000000000000000000000000" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
changed_paths=$(git diff --name-only "$GITHUB_EVENT_BEFORE" "$GITHUB_SHA")
|
||||
else
|
||||
changed_paths=$(git ls-files)
|
||||
fi
|
||||
|
||||
# 防御性兜底:diff 结果为空(异常事件或比较失败)时按全量运行,
|
||||
# 宁可多跑也不能漏测。
|
||||
if [ -z "$changed_paths" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rust=false
|
||||
shell=false
|
||||
while IFS= read -r path; do
|
||||
case "$path" in
|
||||
Cargo.toml|Cargo.lock|rust-toolchain.toml|.cargo/*|*.rs|*/Cargo.toml|*/build.rs|*.sql|.github/workflows/*.yml|.github/workflows/*.yaml)
|
||||
rust=true
|
||||
;;
|
||||
*.sh|*.py|README.md|*/README.md|.env.example|Dockerfile*|docker-compose*.yml|docker-compose*.yaml)
|
||||
shell=true
|
||||
;;
|
||||
esac
|
||||
done <<< "$changed_paths"
|
||||
|
||||
echo "rust=$rust" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=$shell" >> "$GITHUB_OUTPUT"
|
||||
|
||||
shell_security:
|
||||
name: Shell security fixtures
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.shell == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Run installer and supply-chain fixtures
|
||||
shell: bash
|
||||
run: |
|
||||
python3 tests/compose_database_config_test.py
|
||||
bash tests/deploy_state_safety_test.sh
|
||||
bash tests/install_archive_safety_test.sh
|
||||
bash tests/install_container_runtime_security_test.sh
|
||||
bash tests/install_current_release_link_test.sh
|
||||
bash tests/install_local_bundle_safety_test.sh
|
||||
bash tests/install_privileged_write_safety_test.sh
|
||||
bash tests/install_source_trust_test.sh
|
||||
bash tests/release_supply_chain_test.sh
|
||||
bash tests/update_compose_safety_test.sh
|
||||
bash tests/tunnel_installer_config_security_test.sh
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: rustfmt
|
||||
@@ -53,24 +191,28 @@ jobs:
|
||||
|
||||
clippy_gateway:
|
||||
name: Clippy (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# Gateway lint 与 Gateway 测试都可能触发 mold/大型链接依赖,单独隔离缓存
|
||||
# 指纹,避免不同 job 的构建产物互相驱逐或复用错误的链接参数。
|
||||
shared-key: rust-ci-gateway-clippy-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -87,24 +229,26 @@ jobs:
|
||||
|
||||
clippy_data:
|
||||
name: Clippy (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -121,24 +265,26 @@ jobs:
|
||||
|
||||
clippy_rest:
|
||||
name: Clippy (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -157,6 +303,7 @@ jobs:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- clippy_gateway
|
||||
- clippy_data
|
||||
- clippy_rest
|
||||
@@ -164,6 +311,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify clippy jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; clippy jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.clippy_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_rest.result }}" != "success" ]; then
|
||||
@@ -173,82 +324,100 @@ jobs:
|
||||
|
||||
test_gateway:
|
||||
name: Test (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# 构建指纹提到 job 级:mold RUSTFLAGS / 栈 / sccache 对 lib、bins、integration 三步保持一致,
|
||||
# 避免 step 级 env 漂移导致同 job 内 rustc 指纹不一致。
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
# 与 rust-toolchain.toml、fmt/clippy 钉在同一版本,避免浮动 stable 换指纹导致全量重编
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# mold RUSTFLAGS 只在本 job 生效:独立 cache key,避免与无 mold 的 job 互相污染指纹
|
||||
shared-key: rust-ci-gateway-test-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Setup mold
|
||||
uses: rui314/setup-mold@v1
|
||||
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test lib
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --lib
|
||||
|
||||
- name: Test bins
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --bins
|
||||
|
||||
# 只运行独立 integration targets;显式列出目标,避免 --tests 再次执行 lib/bin 测试。
|
||||
- name: Test integration targets
|
||||
run: >-
|
||||
cargo nextest run -p aether-gateway
|
||||
--test admin_unsigned_identity_headers
|
||||
--test architecture_guard
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
test_data:
|
||||
name: Test (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_REQUIRE_LOCAL_POSTGRES_TESTS: "true"
|
||||
run: cargo nextest run -p aether-data
|
||||
|
||||
- name: Show sccache stats
|
||||
@@ -260,29 +429,31 @@ jobs:
|
||||
|
||||
check_data_features:
|
||||
name: Check (Data Feature - ${{ matrix.feature }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
feature:
|
||||
- postgres
|
||||
- mysql
|
||||
- sqlite
|
||||
- all-drivers
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check selected data driver
|
||||
env:
|
||||
@@ -299,27 +470,31 @@ jobs:
|
||||
|
||||
test_rest:
|
||||
name: Test (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
@@ -336,31 +511,33 @@ jobs:
|
||||
|
||||
test_data_adapters:
|
||||
name: Test (Data Adapter - ${{ matrix.package }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
package:
|
||||
- aether-data-postgres
|
||||
- aether-data-mysql
|
||||
- aether-data-sqlite
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test adapter
|
||||
env:
|
||||
@@ -377,21 +554,28 @@ jobs:
|
||||
|
||||
check_integration_scenarios:
|
||||
name: Test (Integration Scenarios)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test scenario binaries and end-to-end suites
|
||||
env:
|
||||
@@ -410,6 +594,7 @@ jobs:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- test_gateway
|
||||
- test_data
|
||||
- check_data_features
|
||||
@@ -420,6 +605,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify test jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; test jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.test_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.test_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.check_data_features.result }}" != "success" ] || \
|
||||
@@ -430,42 +619,10 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
data_db_smoke_sqlite:
|
||||
name: Data DB Smoke (SQLite)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run SQLite data smoke tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: cargo test -p aether-data --all-features sqlite --lib
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_postgres:
|
||||
name: Data DB Smoke (Postgres)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
@@ -482,22 +639,24 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Add PostgreSQL server binaries to PATH
|
||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||
@@ -516,6 +675,20 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_provider_upstream_metadata_migration_preserves_json_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run Postgres payment callback regression tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_payment_callback --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres batch wallet deduction regression
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_bulk_wallet_adjustment_persists_actual_delta_and_skips_zero_ledger --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres API key lifecycle tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -535,112 +708,6 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run SQLite-to-Postgres import smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features sqlite_core_export_reads_migrated_database_rows --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_mysql:
|
||||
name: Data DB Smoke (MySQL)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
env:
|
||||
MYSQL_DATABASE: aether_test
|
||||
MYSQL_USER: aether
|
||||
MYSQL_PASSWORD: aether
|
||||
MYSQL_ROOT_PASSWORD: aether_root
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uaether -paether --silent"
|
||||
--health-interval=5s
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run MySQL migration smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_migrations_create_core_config_tables_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage write smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_write_repository_upserts_and_flushes_counters_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_read_repository_reads_usage_contract_views_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL provider catalog smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_provider_catalog_repository_round_trips_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL core export smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_wallet_read_repository_reads_wallet_contract_views --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet daily usage aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL stats aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
@@ -652,16 +719,21 @@ jobs:
|
||||
name: Data DB Smoke
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- data_db_smoke_sqlite
|
||||
- changes
|
||||
- data_db_smoke_postgres
|
||||
- data_db_smoke_mysql
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify database smoke jobs
|
||||
run: |
|
||||
if [ "${{ needs.data_db_smoke_sqlite.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_postgres.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_mysql.result }}" != "success" ]; then
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; database smoke jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||
echo "Data DB smoke failed"
|
||||
exit 1
|
||||
fi
|
||||
@@ -670,18 +742,40 @@ jobs:
|
||||
name: check
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- fmt
|
||||
- clippy
|
||||
- test
|
||||
- data_db_smoke
|
||||
- shell_security
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify required jobs
|
||||
run: |
|
||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||
[ "${{ needs.test.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
||||
# changes 失败或未产出 scope 时不允许直接放行,避免假绿。
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
rust="${{ needs.changes.outputs.rust }}"
|
||||
shell="${{ needs.changes.outputs.shell }}"
|
||||
|
||||
if [ "$rust" != "true" ] && [ "$shell" != "true" ]; then
|
||||
echo "No Rust or shell scope changed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$rust" = "true" ] && {
|
||||
[ "${{ needs.fmt.result }}" != "success" ] ||
|
||||
[ "${{ needs.clippy.result }}" != "success" ] ||
|
||||
[ "${{ needs.test.result }}" != "success" ] ||
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ];
|
||||
}; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$shell" = "true" ] && [ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -13,6 +13,10 @@
|
||||
.plans
|
||||
.playwright-mcp/
|
||||
|
||||
docs/architecture
|
||||
!docs/architecture/architecture-dark.svg
|
||||
!docs/architecture/architecture-light.svg
|
||||
|
||||
### Python ###
|
||||
*.db
|
||||
*.db-*
|
||||
|
||||
Generated
+608
-387
File diff suppressed because it is too large
Load Diff
+4
-5
@@ -11,8 +11,6 @@ members = [
|
||||
"crates/aether-routing-core",
|
||||
"crates/aether-data/contracts",
|
||||
"crates/aether-data/adapters/postgres",
|
||||
"crates/aether-data/adapters/mysql",
|
||||
"crates/aether-data/adapters/sqlite",
|
||||
"crates/aether-data/query",
|
||||
"crates/aether-data/schema",
|
||||
"crates/aether-dispatch-core",
|
||||
@@ -66,8 +64,6 @@ aether-provider-pool = { path = "crates/aether-provider/pool" }
|
||||
aether-routing-core = { path = "crates/aether-routing-core" }
|
||||
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
||||
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
||||
aether-data-mysql = { path = "crates/aether-data/adapters/mysql" }
|
||||
aether-data-sqlite = { path = "crates/aether-data/adapters/sqlite" }
|
||||
aether-data-query = { path = "crates/aether-data/query" }
|
||||
aether-data-schema = { path = "crates/aether-data/schema" }
|
||||
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
||||
@@ -96,11 +92,13 @@ aether-usage-core = { path = "crates/aether-usage/core" }
|
||||
aether-usage-runtime = { path = "crates/aether-usage/runtime" }
|
||||
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
|
||||
aether-gateway = { path = "apps/aether-gateway" }
|
||||
aether-tunnel = { path = "apps/aether-tunnel" }
|
||||
aether-http = { path = "crates/aether-http" }
|
||||
aether-runtime = { path = "crates/aether-runtime/base" }
|
||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
axum = "0.8"
|
||||
@@ -117,8 +115,9 @@ flate2 = "1"
|
||||
futures-util = "0.3"
|
||||
hmac = "0.12"
|
||||
http = "1"
|
||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
||||
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||
percent-encoding = "2"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||
regex = "1"
|
||||
|
||||
+9
-4
@@ -10,20 +10,24 @@
|
||||
|
||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||
# distroless has no shell, so we use busybox to set up the symlink.
|
||||
FROM busybox:1.37-musl AS layout
|
||||
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||
|
||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||
|
||||
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||
# identity can traverse and read every packaged asset.
|
||||
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
|
||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||
|
||||
# --- final stage: distroless runtime ---
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||
|
||||
COPY --from=layout /opt/aether /opt/aether
|
||||
|
||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
||||
|
||||
ENV RUST_LOG=aether_gateway=info \
|
||||
APP_PORT=8084 \
|
||||
HOME=/tmp/aether-home \
|
||||
AETHER_UPDATE_STRATEGY=docker \
|
||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||
|
||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER root
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -157,4 +157,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
||||
|
||||
@@ -156,4 +156,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -36,15 +36,6 @@ lowercase() {
|
||||
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
dev_uses_sqlite_database() {
|
||||
local driver
|
||||
local url
|
||||
driver="$$(lowercase "$${AETHER_DATABASE_DRIVER:-}")"
|
||||
url="$${AETHER_DATABASE_URL:-$${DATABASE_URL:-}}"
|
||||
|
||||
[[ "$${driver}" == "sqlite" || "$${url}" == sqlite:* ]]
|
||||
}
|
||||
|
||||
dev_uses_postgres_database() {
|
||||
local driver
|
||||
local url
|
||||
@@ -68,9 +59,6 @@ dev_uses_redis_runtime() {
|
||||
if [[ "$${backend}" == "redis" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if dev_uses_sqlite_database; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -356,6 +344,9 @@ if ! ensure_dev_infra; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=> 编译 aether-gateway..."
|
||||
cargo build -p aether-gateway --bin aether-gateway
|
||||
|
||||
GATEWAY_PID=""
|
||||
GATEWAY_LOG_DIR=""
|
||||
GATEWAY_LOG_FILE=""
|
||||
@@ -364,8 +355,8 @@ create_gateway_log_file
|
||||
|
||||
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
||||
echo "=> 日志过滤: $${RUST_LOG}"
|
||||
echo "=> 执行命令: cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT}"
|
||||
cargo run -p aether-gateway --bin aether-gateway -- --app-port "$${APP_PORT}" > >(
|
||||
echo "=> 执行命令: target/debug/aether-gateway --app-port $${APP_PORT}"
|
||||
target/debug/aether-gateway --app-port "$${APP_PORT}" > >(
|
||||
tee -a "$${GATEWAY_LOG_FILE}"
|
||||
) 2>&1 &
|
||||
GATEWAY_PID=$$!
|
||||
@@ -456,7 +447,7 @@ if [ -f .env ]; then
|
||||
fi
|
||||
export APP_PORT="$${APP_PORT:-8084}"
|
||||
|
||||
echo "=> 启动后端: RUST_LOG=$${DEV_RUST_LOG} cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT:-8084}"
|
||||
echo "=> 启动后端: 先编译 aether-gateway,再运行 target/debug/aether-gateway --app-port $${APP_PORT:-8084}"
|
||||
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
||||
backend_pid=$$!
|
||||
|
||||
|
||||
@@ -44,86 +44,39 @@ cd Aether
|
||||
|
||||
# 2. 配置环境变量
|
||||
cp .env.example .env
|
||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
||||
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||
chmod 600 .env
|
||||
# 生成 JWT / 加密 / Postgres / Redis 独立随机密钥,并填入 .env
|
||||
./generate_keys.sh
|
||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||
|
||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
||||
# Postgres + Redis (适用于企业或多人使用)
|
||||
# 3. Docker 部署 / 更新(PostgreSQL + Redis)
|
||||
docker compose pull && docker compose up -d
|
||||
# Single Node (适用于个人用户或朋友分享)
|
||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
||||
```
|
||||
|
||||
### 一键更新
|
||||
|
||||
Docker Compose 部署后,可在部署目录直接执行:
|
||||
|
||||
```bash
|
||||
./update.sh
|
||||
```
|
||||
|
||||
`update.sh` 会拉取最新 `app` 镜像并重建 `app` 容器,Docker named volumes、`./data` 和 `./logs` 不会被删除。Single Node 部署也可显式指定:
|
||||
|
||||
```bash
|
||||
./update.sh --mode single-node
|
||||
```
|
||||
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
||||
|
||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
||||
|
||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
||||
|
||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
||||
|
||||
如果是本地源码构建镜像的部署,继续使用:
|
||||
|
||||
```bash
|
||||
./deploy.sh
|
||||
```
|
||||
|
||||
如果要在本机联调“管理后台在线更新”本身,可启动仓库内置的 release-layout 测试环境:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.release-local.yml up -d --build
|
||||
```
|
||||
|
||||
这套环境会用当前源码构建一个本地测试镜像,但编译为 `release` 类型,并默认伪装成 `v0.7.0`,这样后台会按正式发布版逻辑开放“立即更新”。默认监听 `http://127.0.0.1:18085`,数据目录使用 `./data-release-local`;日志默认走 `docker logs`,不会影响你正在跑的源码构建容器。
|
||||
|
||||
如果这套容器在 `prepare-update` 时访问 GitHub 失败,而你本机是通过代理出网,请在 `.env` 里把 `AETHER_UPDATE_PROXY_URL` 写成宿主机地址,例如 `http://host.docker.internal:7890`;容器内的 `127.0.0.1` 指向容器自身,不是宿主机。
|
||||
|
||||
如果想重置这套联调环境(包括 `/opt/aether/current` 和已下载的历史版本),执行:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.release-local.yml down -v
|
||||
```
|
||||
|
||||
可选变量:
|
||||
|
||||
- `AETHER_RELEASE_LOCAL_VERSION`:本地联调镜像对外声明的当前版本,默认 `v0.7.0`
|
||||
- `AETHER_RELEASE_LOCAL_PORT`:本地联调端口,默认 `18085`
|
||||
- `LOCAL_RELEASE_APP_IMAGE`:本地联调镜像名,默认 `aether-app:release-local`
|
||||
|
||||
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
||||
### 一键安装(PostgreSQL + Redis)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/fawney19/Aether.git
|
||||
cd Aether
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose
|
||||
```
|
||||
|
||||
正式版和 Nightly 自动构建仅提供 Linux `amd64` / `arm64` 二进制包,Docker 镜像同样支持这两种架构。macOS 用户可使用 Docker 或自行从源码构建;安装脚本保留对历史 macOS 制品的兼容。独立 Aether Tunnel 的多平台发行不受此调整影响。
|
||||
|
||||
原生 Linux systemd 安装需先准备 PostgreSQL,将连接串通过 `DATABASE_URL` 传给安装进程,并选择 `--mode single-node`;不再自动创建本地数据库文件。
|
||||
|
||||
### Nightly(每日 main 构建)
|
||||
|
||||
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
||||
|
||||
安装最新 nightly(Linux systemd / macOS launchd + SQLite):
|
||||
安装最新 nightly(PostgreSQL + Redis):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --channel nightly
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose --channel nightly
|
||||
```
|
||||
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制方式可重新执行上述安装命令升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制部署请沿用已有 PostgreSQL 环境配置,并使用 `--mode single-node --channel nightly` 重新运行安装脚本升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
|
||||
## 本地开发
|
||||
|
||||
@@ -136,7 +89,7 @@ make dev
|
||||
|
||||
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
||||
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
||||
数据库 schema 和历史数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。排查或部署前预执行时可使用:
|
||||
`make dev` 会先完成后端编译,再开始计算服务健康检查超时。数据库 schema 和必要的派生数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。升级不会主动重写或清除已有业务历史记录,新写入会直接遵循当前的数据持久化策略。排查或部署前预执行时可使用:
|
||||
|
||||
```bash
|
||||
make db-status
|
||||
@@ -168,26 +121,56 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
||||
## 环境变量
|
||||
|
||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||
- `DATABASE_URL`:数据库连接串;SQLite 例如 `sqlite:///opt/aether/data/aether.db`,Postgres 例如 `postgresql://postgres:aether@postgres:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS` / `AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS`:普通数据库连接的单条 SQL / 锁等待期限,默认 `30000` / `3000` 毫秒,显式 `0` 关闭;不是整个事务总期限。迁移与历史 backfill 使用独立连接放宽,事务可通过局部设置覆盖
|
||||
- `AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES`:usage 诊断正文共享预算,默认 `134217728`(128 MiB),按 JSON 堆内存估算,覆盖进入终态队列的 seed、Redis 解码后的事件、数据库写入 DTO 及其正文副本。额度不足或显式 `0` 时先保留计费事实,再舍弃诊断正文;已有清空或禁用状态保持不变,其余标记截断。预算随正文保留到释放,后台构建或压缩不会因调用方取消而提前归还额度。该额度不覆盖原始 Redis 批次、解码临时分配、序列化及压缩结果、协议观察缓冲或进程总内存;可通过 `usage_runtime_event_capture_memory_*` 指标观察
|
||||
- `AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES`:新增 usage 队列消息的完整 JSON payload 上限,默认 `1048576`(1 MiB),按序列化后的 UTF-8 字节计算,显式 `0` 非法。超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义时拒绝入队,终态消息尝试受限数据库落库,失败则明确失败,不继续 Redis 重试。该限制不覆盖存量 Redis 消息、整个读取批次、DLQ 或进程总内存。`usage_runtime_queue_payload_*` 导出上限及进程级降级、拒绝编码尝试次数,包含入队和重试预校验,不代表唯一事件数;`usage_runtime_enqueue_retry_permanent_failure_total` 记录永久输入错误导致的重试拒绝或终止
|
||||
- `AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES` / `AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES`:usage worker 读取和重领共用的进程级逻辑 payload 预留,默认总额 `134217728`(128 MiB)、单批目标 `8388608`(8 MiB)。按当前 `QUEUE_PAYLOAD_MAX_BYTES` 推导实际 COUNT,默认最多读取 8 条,自动扩容使用实际 COUNT 判断批次是否读满。预留覆盖读取、整批处理和确认,额度不足等待;取消/失败释放。单批目标至少允许一条,当前 payload 上限大于总额时读取报配置错误。`0` 或非法值回退默认,过大值收敛到约 4 GiB 的有效总额。收到消息后按全部字段值长度缩减多余预留;历史消息、其他生产者使用更高上限或额外字段可能超出估算,仍继续原计费流程并记录 `usage_runtime_queue_read_oversized_*`。`usage_runtime_queue_read_*` 同时导出预留、等待与累计字段字节;该预留不是 RESP 解码、连接缓冲容量、字段结构、诊断 JSON、DLQ 或进程 RSS 的硬上限,旧公开 Vec 读取接口不携带处理阶段预留
|
||||
- `AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES` / `AETHER_USAGE_DLQ_ENCODING_MAX_JOBS`:死信原文和 JSON 编码独立共享预留,默认 `67108864`(64 MiB)、最多 `4` 个后台编码及写入任务。根据原始字段、ID、错误字符串及 JSON 最坏 6 倍转义一次预留;预算占满或单条超总额时立即失败,worker 保留原消息等待重领,不截断账务原文。编码失败会继续处理同批其他消息,只确认成功项,批次末尾仍报告失败;存储转移失败则停止该批后续处理。取消编码等待不会提前归还仍在后台使用的额度。`0`/非法值回退默认,bytes 最大约 4 GiB,jobs 最大 128;超大存量消息可能需要调高总额后恢复。`usage_runtime_dlq_encoding_*` 导出额度、在途任务、拒绝和编码尝试次数;不包含字段结构、字符串额外容量、Redis 命令/连接副本或进程 RSS。内置 Redis/Memory worker 将死信追加、源 ACK 和删除作为一次原子转移,同一源 stream、消费组及 pending ID 的并发或重试只追加一次;Redis 要求 7+ 及 `EVAL/TYPE/XPENDING/XADD/XACK/XDEL` 权限,Cluster 两键须同 slot(当前默认键不自动迁移)。源和 DLQ 不能同名。源已不在 PEL 时不宣称已归档;外部 ACK/trim/delete 及多消费组仍有原来的删除语义。公开 `push_dead_letter` 仍为追加接口,未实现新原子 trait 方法的外部后端沿用追加后 ACK,仍可能重复归档
|
||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_GATEWAY_MAX_HTTP_CONNECTIONS`:二进制入口全部监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和 HTTP 升级后仍存活的 socket。未设置或 `0` 时使用请求上限与 WebSocket 上限之和;自动及显式值均最多 `65536`,已知 FD soft limit 时进一步限制为 `max(1, (FD - 256) / 2)`。接入后立即尝试取得额度,满额时关闭新连接,不创建 HTTP 处理任务、不等待额度,不返回 HTTP 状态码;取消、解析失败和连接释放归还,WebSocket 升级不会提前归还。HTTP/2 多流共用一个 TCP 许可,原请求和 WebSocket 准入仍独立有效。`gateway_http_connections_*` 导出配置上限、当前数、高水位、拒绝数及 accept 错误数。该限制不包含 kernel backlog、上游、Redis 或数据库连接,也不是整个进程 FD/内存硬上限。临时 accept 错误重试,资源类错误退避一秒后重试,避免单次错误停止监听
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`;压缩和未知长度上传按实际缓冲增长申请额度,解压时计入同时存活的输入和输出。额度不足返回 `503`;接近单请求上限的压缩上传需要为输入和解压输出预留额外预算
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`;显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||
- `AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS`:上游流首包后的空闲超时,默认 `300000ms`;请求执行配置中的 `read_ms` 优先,显式 `0` 关闭对应超时。网关生成的 keepalive 不会重置计时
|
||||
- `AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES`:进程内流式响应诊断捕获的共享字节预算,默认 `134217728`(128 MiB);包含 provider/client 捕获容量和扩容时的新旧分配。额度不足时仅截断审计副本,显式 `0` 关闭此类捕获;协议解析、客户端传输和计费观察继续执行。该预算不包含协议解析缓冲、终态编码及 usage 队列副本,不是进程总内存上限
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||
- `AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS`:跨网关 owner relay 解析到私有/保留地址时的显式运维开关,默认关闭;仅当多网关 relay URL 是受控的内网 HTTPS 地址时设置为 `true`。它不改变普通 provider 请求的 DNS/代理策略,也不允许明文 HTTP 非 loopback relay
|
||||
- `AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST`:更窄的 owner relay 私网例外,填写逗号分隔的精确主机名(例如 `gateway-a.internal,gateway-b.internal`,忽略大小写和末尾点);仅这些主机解析出的私有地址会被允许,并且请求仍使用解析后地址 pin。不要填写通配符或 `.internal` 这类后缀
|
||||
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。配置 Redis 时使用 `redis`,否则使用 `memory`;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
||||
- `DB_PASSWORD` / `REDIS_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||
|
||||
运行日志由独立后台线程写入 stdout 和文件,每个输出队列最多 4096 条、保留正文最多 8 MiB(包含正在写入的记录),单条最多 256 KiB。队列满、正文预算不足或单条超限时整条丢弃,不等待日志设备;`Both` 两个输出独立降级。`logging_stdout_*` 和 `logging_file_*` 指标记录丢弃和写入错误,网关指标沿用其命名空间前缀。正常退出时日志最多等待 2 秒排空;这不是请求优雅排空或整个进程退出期限。日志格式化仍在调用线程执行,日志预算不包含格式化临时内存,运行日志也不能作为可靠计费账本。
|
||||
|
||||
### S3 备份离线恢复
|
||||
|
||||
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||
|
||||
```bash
|
||||
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||
--input ./backup.json.zst.aes256gcm \
|
||||
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||
--output ./restored-backup.json
|
||||
```
|
||||
|
||||
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||
|
||||
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ npm --prefix aether-vscodex/web run build
|
||||
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
||||
共享连接目录前横向扩容 sidecar。
|
||||
|
||||
登录 Aether 后打开“Codex 远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
登录 Aether 后打开“远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
||||
VS Code SecretStorage,并同时保持本机控制台连接。
|
||||
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
test binary
|
||||
@@ -62,17 +62,18 @@ flate2.workspace = true
|
||||
futures-util.workspace = true
|
||||
hmac.workspace = true
|
||||
http.workspace = true
|
||||
http-body = "1"
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
||||
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||
libc = "0.2"
|
||||
md-5 = "0.10"
|
||||
object_store.workspace = true
|
||||
parking_lot = "0.12"
|
||||
percent-encoding.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
rsa = "0.9.10"
|
||||
rustls.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -81,7 +82,7 @@ sha1 = "0.10"
|
||||
sha2 = { workspace = true, features = ["oid"] }
|
||||
socket2.workspace = true
|
||||
tar.workspace = true
|
||||
sqlx = { workspace = true, features = ["postgres", "mysql", "sqlite", "migrate"] }
|
||||
sqlx = { workspace = true, features = ["postgres", "migrate"] }
|
||||
sysinfo = "0.32"
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -102,5 +103,7 @@ tikv-jemallocator = { version = "0.6", optional = true }
|
||||
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aether-testkit = { workspace = true, features = ["postgres"] }
|
||||
aether-test-support.workspace = true
|
||||
aws-lc-rs.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
@@ -30,7 +30,7 @@ struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||
default_value = "/tmp/aether-execution-runtime.sock"
|
||||
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||
)]
|
||||
unix_socket: PathBuf,
|
||||
|
||||
@@ -71,8 +71,13 @@ struct Args {
|
||||
distributed_request_command_timeout_ms: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||
run()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
init_service_runtime(ServiceRuntimeConfig::new(
|
||||
|
||||
@@ -88,8 +88,13 @@ struct Args {
|
||||
distributed_request_command_timeout_ms: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||
run()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
init_service_runtime(ServiceRuntimeConfig::new(
|
||||
"aether-tunnel-standalone",
|
||||
"aether_gateway=info",
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
pub(crate) use crate::handlers::admin::{
|
||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
||||
reconcile_admin_fixed_provider_template_endpoints,
|
||||
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
||||
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
||||
OAUTH_REQUEST_FAILED_PREFIX,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||
};
|
||||
|
||||
use crate::handlers::admin::{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||
use base64::Engine as _;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
||||
}
|
||||
|
||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let Some(normalized_bytes) =
|
||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||
else {
|
||||
|
||||
@@ -69,9 +69,14 @@ pub(crate) use aether_ai_formats::api::{
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::protocol::stream::CanonicalUsage as StreamingCanonicalUsage;
|
||||
/// Codex client identity headers re-exported for out-of-crate probe binaries,
|
||||
/// which must reach `aether_ai_formats` through this seam.
|
||||
pub use aether_ai_formats::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
/// Codex client identity accessors re-exported for out-of-crate probe binaries,
|
||||
/// which must reach the runtime profile through this seam.
|
||||
pub use aether_ai_formats::{codex_client_originator, codex_client_user_agent};
|
||||
/// Codex 动态客户端画像 API 只允许经此根缝进入 gateway,避免其它模块直接依赖 formats crate。
|
||||
pub(crate) use aether_ai_formats::{
|
||||
codex_client_profile, codex_client_version, set_codex_cli_version, set_codex_client_profile,
|
||||
CodexClientProfile,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::{CODEX_RESPONSES_LITE_HEADER, UPSTREAM_IS_STREAM_KEY};
|
||||
|
||||
pub(crate) fn parse_direct_request_body(
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_provider_transport::CodexFingerprintConvergenceContext;
|
||||
use http::{request::Parts, HeaderMap};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::client_session_affinity::codex_request_signals_from_request;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<CodexFingerprintConvergenceContext>>);
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<ProviderOutboundRequestContext>>);
|
||||
|
||||
impl Default for CodexFingerprintContextSlot {
|
||||
fn default() -> Self {
|
||||
@@ -18,11 +18,7 @@ impl Default for CodexFingerprintContextSlot {
|
||||
}
|
||||
|
||||
impl CodexFingerprintContextSlot {
|
||||
fn resolve(
|
||||
&self,
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
fn resolve(&self, headers: &HeaderMap, body_json: &Value) -> ProviderOutboundRequestContext {
|
||||
self.0
|
||||
.get_or_init(|| {
|
||||
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
||||
@@ -34,10 +30,10 @@ impl CodexFingerprintContextSlot {
|
||||
pub(crate) fn resolve_codex_fingerprint_context(
|
||||
parts: &Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
if let Some(context) = parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.cloned()
|
||||
{
|
||||
return context;
|
||||
@@ -51,7 +47,7 @@ pub(crate) fn resolve_codex_fingerprint_context(
|
||||
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
&& parts
|
||||
.extensions
|
||||
@@ -67,11 +63,11 @@ pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
pub(crate) fn ensure_codex_fingerprint_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context = resolve_codex_fingerprint_context(parts, body_json);
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
{
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -84,7 +80,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
logical_turn_id: &str,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context =
|
||||
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -94,7 +90,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
|
||||
pub(crate) fn restore_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
context: &CodexFingerprintConvergenceContext,
|
||||
context: &ProviderOutboundRequestContext,
|
||||
) {
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
parts.extensions.insert(context.clone());
|
||||
@@ -104,10 +100,9 @@ fn build_codex_fingerprint_context(
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
logical_turn_id: String,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
||||
let mut context =
|
||||
CodexFingerprintConvergenceContext::new(logical_turn_id, current_unix_millis());
|
||||
let mut context = ProviderOutboundRequestContext::new(logical_turn_id, current_unix_millis());
|
||||
|
||||
if let Some(turn_id) = signals.turn_id {
|
||||
context = context.with_original_turn_id(turn_id);
|
||||
@@ -160,14 +155,14 @@ mod tests {
|
||||
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
||||
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
||||
assert_eq!(
|
||||
parts.extensions.get::<CodexFingerprintConvergenceContext>(),
|
||||
parts.extensions.get::<ProviderOutboundRequestContext>(),
|
||||
Some(&context)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_context_wins_over_retry_request_signals() {
|
||||
let original = CodexFingerprintConvergenceContext::new("logical-turn", 1234)
|
||||
let original = ProviderOutboundRequestContext::new("logical-turn", 1234)
|
||||
.with_original_turn_id("original-turn")
|
||||
.with_original_client_session_id("original-thread")
|
||||
.with_original_prompt_cache_key("original-cache");
|
||||
|
||||
@@ -18,6 +18,12 @@ pub(crate) fn maybe_build_local_stream_rewriter<'a>(
|
||||
}
|
||||
|
||||
impl LocalStreamRewriter<'_> {
|
||||
pub(crate) fn into_owned(self) -> LocalStreamRewriter<'static> {
|
||||
LocalStreamRewriter {
|
||||
inner: self.inner.into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_chunk(&mut self, chunk: &[u8]) -> Result<Vec<u8>, GatewayError> {
|
||||
self.inner.push_chunk(chunk).map_err(map_surface_error)
|
||||
}
|
||||
|
||||
@@ -11,10 +11,10 @@ use super::{
|
||||
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
||||
maybe_build_local_core_sync_finalize_response,
|
||||
};
|
||||
use crate::ai_serving::GatewayControlDecision;
|
||||
use crate::ai_serving::{
|
||||
convert_openai_chat_response_to_openai_responses,
|
||||
convert_openai_responses_response_to_openai_chat,
|
||||
convert_openai_responses_response_to_openai_chat, openai_responses_message_item_id,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::usage::GatewaySyncReportRequest;
|
||||
|
||||
@@ -192,7 +192,7 @@ fn aggregates_openai_responses_stream_completed_event_to_final_response() {
|
||||
"output_text": "Hello",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -843,7 +843,7 @@ fn converts_claude_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Claude CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -907,7 +907,7 @@ fn converts_claude_cli_tool_use_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -977,7 +977,7 @@ fn converts_gemini_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Gemini CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1046,7 +1046,7 @@ fn converts_gemini_cli_function_call_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "resp_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1252,7 +1252,7 @@ fn local_finalize_handles_openai_responses_openai_family_sync_response_even_when
|
||||
"model": "gpt-5",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_family_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_family_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
|
||||
@@ -996,7 +996,7 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
||||
);
|
||||
|
||||
if page_is_exact_auth_api_key_concurrency_limited(&page) {
|
||||
if self.wait_for_auth_api_key_concurrency_retry().await {
|
||||
if self.wait_for_auth_api_key_concurrency_retry().await? {
|
||||
continue;
|
||||
}
|
||||
self.persist_final_auth_api_key_concurrency_skips(page.skipped_candidates)
|
||||
@@ -1087,20 +1087,23 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> bool {
|
||||
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> Result<bool, GatewayError> {
|
||||
let now = Instant::now();
|
||||
let deadline = *self
|
||||
.auth_api_key_concurrency_wait_deadline
|
||||
.get_or_insert(now + AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||
if now >= deadline {
|
||||
return false;
|
||||
if !crate::scheduler::candidate::wait_for_auth_api_key_concurrency_retry(
|
||||
self.state.app(),
|
||||
Some(&self.auth_snapshot),
|
||||
deadline,
|
||||
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let sleep_duration =
|
||||
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY.min(deadline.saturating_duration_since(now));
|
||||
tokio::time::sleep(sleep_duration).await;
|
||||
self.page_cursor.restart_scan();
|
||||
true
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn persist_final_auth_api_key_concurrency_skips(
|
||||
@@ -2289,6 +2292,103 @@ mod tests {
|
||||
candidate
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auth_concurrency_wait_paged_scan_retries_once_at_original_deadline() {
|
||||
let now = current_unix_ms();
|
||||
let active = serde_json::from_value(json!({
|
||||
"id": "active-candidate",
|
||||
"request_id": "active-request",
|
||||
"api_key_id": "api-key-1",
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"status": "pending",
|
||||
"is_cached": false,
|
||||
"created_at_unix_ms": now,
|
||||
"started_at_unix_ms": now
|
||||
}))
|
||||
.expect("active candidate should build");
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::seed([active]));
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_request_candidate_repository_for_tests(repository),
|
||||
);
|
||||
let mut auth_snapshot = sample_auth_snapshot();
|
||||
auth_snapshot.api_key_concurrent_limit = Some(1);
|
||||
let page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&crate::system_features::ModelDirectivePolicySnapshot::default(),
|
||||
"openai:chat",
|
||||
"gpt-5",
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModel,
|
||||
true,
|
||||
Some("trace-auth-wait"),
|
||||
)
|
||||
.await;
|
||||
let mut cursor = RequestedModelAttemptPageCursor {
|
||||
state: PlannerAppState::new(&app),
|
||||
trace_id: "trace-auth-wait".to_string(),
|
||||
client_api_format: "openai:chat".to_string(),
|
||||
requested_model: "gpt-5".to_string(),
|
||||
auth_snapshot,
|
||||
client_session_affinity: None,
|
||||
required_capabilities: None,
|
||||
routing_policy: None,
|
||||
sticky_session_token: None,
|
||||
request_auth_channel: None,
|
||||
skipped_user_id: "user-1".to_string(),
|
||||
skipped_api_key_id: "api-key-1".to_string(),
|
||||
skipped_required_capabilities: None,
|
||||
skipped_error_context: "test auth wait",
|
||||
record_runtime_miss_diagnostic: false,
|
||||
resolution_mode: LocalCandidateResolutionMode::Standard,
|
||||
decorate_skipped_candidate: Arc::new(identity_skipped_candidate),
|
||||
page_cursor,
|
||||
pending_items: VecDeque::new(),
|
||||
skipped_provider_ids: BTreeSet::new(),
|
||||
skipped_endpoint_ids: BTreeSet::new(),
|
||||
skipped_credential_ids: BTreeSet::new(),
|
||||
candidate_count: 0,
|
||||
next_candidate_index: 0,
|
||||
remembered_affinity: false,
|
||||
scheduler_cache_affinity_enabled: false,
|
||||
auth_api_key_concurrency_wait_deadline: None,
|
||||
deferred_error: None,
|
||||
};
|
||||
|
||||
let started = Instant::now();
|
||||
let mut scan_restarts = 0;
|
||||
while cursor
|
||||
.wait_for_auth_api_key_concurrency_retry()
|
||||
.await
|
||||
.expect("auth wait should succeed")
|
||||
{
|
||||
scan_restarts += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
scan_restarts, 1,
|
||||
"blocked polls must not restart page scans"
|
||||
);
|
||||
assert!(started.elapsed() >= AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||
let original_deadline = cursor.auth_api_key_concurrency_wait_deadline;
|
||||
assert!(!cursor
|
||||
.wait_for_auth_api_key_concurrency_retry()
|
||||
.await
|
||||
.expect("expired auth wait should succeed"));
|
||||
assert_eq!(
|
||||
cursor.auth_api_key_concurrency_wait_deadline,
|
||||
original_deadline
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_group_keys_are_not_persisted_as_available_before_attempt() {
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
@@ -2325,16 +2425,11 @@ mod tests {
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[0].candidate_index, 2);
|
||||
assert_eq!(
|
||||
stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2482,14 +2577,23 @@ mod tests {
|
||||
|
||||
assert!(should_cache_resolved_candidate_page(&cursor));
|
||||
|
||||
let fixed_order_app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"scheduling_mode".to_string(),
|
||||
json!("fixed_order"),
|
||||
)]),
|
||||
);
|
||||
let fixed_order_app = AppState::new().expect("state should build");
|
||||
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||
group_id: Some("routing-group-fixed-order".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
requested_model: "gpt-5".to_string(),
|
||||
resolved_model: "gpt-5".to_string(),
|
||||
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
matched_rules: Vec::new(),
|
||||
};
|
||||
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&fixed_order_app),
|
||||
&model_directive_policy,
|
||||
@@ -2499,7 +2603,7 @@ mod tests {
|
||||
true,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
Some(&fixed_order_policy),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
@@ -2517,7 +2621,7 @@ mod tests {
|
||||
auth_snapshot,
|
||||
client_session_affinity: None,
|
||||
required_capabilities: None,
|
||||
routing_policy: None,
|
||||
routing_policy: Some(fixed_order_policy),
|
||||
sticky_session_token: None,
|
||||
request_auth_channel: None,
|
||||
skipped_user_id: "user-1".to_string(),
|
||||
@@ -2615,16 +2719,11 @@ mod tests {
|
||||
);
|
||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[1].candidate_index, 1);
|
||||
assert_eq!(
|
||||
stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2694,7 +2793,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("ranking metadata should persist as object extra data");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
@@ -2707,14 +2806,7 @@ mod tests {
|
||||
Some(&json!("cached_affinity"))
|
||||
);
|
||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||
assert_eq!(
|
||||
extra_data
|
||||
.get("dispatch_ref")
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("ranked-key"))
|
||||
);
|
||||
assert!(extra_data.get("dispatch_ref").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3052,7 +3144,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("skipped ranking metadata should persist");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
|
||||
@@ -278,13 +278,21 @@ mod tests {
|
||||
|
||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||
Value::Bool(true)
|
||||
|
||||
@@ -251,6 +251,7 @@ mod tests {
|
||||
use aether_ai_serving::{
|
||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||
};
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::{
|
||||
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||
routing_profiles::InMemoryRoutingGroupRepository,
|
||||
@@ -566,6 +567,15 @@ mod tests {
|
||||
api_formats: Option<serde_json::Value>,
|
||||
allowed_models: Option<serde_json::Value>,
|
||||
) -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -577,7 +587,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
api_formats,
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:chat": 1})),
|
||||
@@ -691,7 +701,7 @@ mod tests {
|
||||
let observed_at_unix_secs = current_unix_secs();
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
@@ -700,6 +710,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-b".to_string(),
|
||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||
tunnel_generation: "test-generation-remote".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -710,6 +721,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-a".to_string(),
|
||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||
tunnel_generation: "test-generation-local".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -768,7 +780,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -821,7 +833,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"scheduling_mode".to_string(),
|
||||
@@ -878,7 +890,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -953,7 +965,7 @@ mod tests {
|
||||
.expect("routing strategy should be created");
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_routing_group_repository_for_tests(routing_repository);
|
||||
let state = AppState::new()
|
||||
@@ -1015,7 +1027,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"provider_priority_mode".to_string(),
|
||||
@@ -1081,7 +1093,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1134,7 +1146,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1208,7 +1220,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1288,7 +1300,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1364,7 +1376,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1431,7 +1443,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1514,7 +1526,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1579,7 +1591,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1668,7 +1680,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1754,7 +1766,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1851,7 +1863,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1956,7 +1968,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -2050,7 +2062,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
|
||||
@@ -93,6 +93,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
&self,
|
||||
candidate: Self::Candidate,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason = "transport_snapshot_missing",
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason: "transport_snapshot_missing",
|
||||
@@ -145,6 +155,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
transport: Self::Transport,
|
||||
skip_reason: &'static str,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason,
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason,
|
||||
|
||||
@@ -1473,6 +1473,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::AppState;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::DataLayerError;
|
||||
@@ -2174,6 +2175,19 @@ mod tests {
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
row.provider_id.as_str(),
|
||||
row.key_id.as_str(),
|
||||
"plain-upstream-key",
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
row.key_id.clone(),
|
||||
row.provider_id.clone(),
|
||||
@@ -2185,7 +2199,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -2540,7 +2554,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key");
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
@@ -2660,7 +2674,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key")
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
// Legacy keys deliberately disagree with the routing policy: the
|
||||
// resolved policy must be the only source of scheduler ordering.
|
||||
.with_system_config_values_for_tests([
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||
};
|
||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
let Some(user_id) = decision
|
||||
.auth_context
|
||||
.as_ref()
|
||||
.filter(|auth_context| auth_context.access_allowed)
|
||||
.map(|auth_context| auth_context.user_id.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
|
||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
let Some(action) = state
|
||||
.video_tasks
|
||||
.prepare_openai_content_stream_action_for_user(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
user_id,
|
||||
)
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
|
||||
let auth_context = resolve_execution_runtime_auth_context(
|
||||
state,
|
||||
decision,
|
||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
)
|
||||
.await?;
|
||||
let Some(auth_context) = auth_context else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
&auth_context.user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||
plan_kind,
|
||||
parts.uri.path(),
|
||||
Some(body_json),
|
||||
Some(&auth_context),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
downstream_path = %parts.uri.path(),
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
client_api_format = %plan.client_api_format,
|
||||
upstream_base_url = ?upstream_base_url,
|
||||
upstream_url = %plan.url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
"gateway built local video follow-up sync decision payload"
|
||||
);
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
||||
use crate::ai_serving::transport::CodexFingerprintConvergenceContext;
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::ai_serving::{
|
||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||
"routing provider request violates provider contract";
|
||||
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||
"invalid provider request headers in routing mutation";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||
@@ -56,7 +60,7 @@ pub(crate) struct LocalRequestedModelDecisionInput {
|
||||
pub(crate) client_surface: Option<ClientSurface>,
|
||||
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
||||
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
||||
pub(crate) codex_fingerprint_context: Option<CodexFingerprintConvergenceContext>,
|
||||
pub(crate) provider_outbound_context: Option<ProviderOutboundRequestContext>,
|
||||
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
||||
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
||||
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
||||
@@ -168,7 +172,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -231,7 +235,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -312,10 +316,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
)
|
||||
}
|
||||
}
|
||||
.map_err(|violation| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
||||
})?;
|
||||
.map_err(|_| invalid_routing_provider_contract())?;
|
||||
}
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
@@ -357,7 +358,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
if original_provider_request_body.is_some() {
|
||||
decision.provider_request_body = Some(provider_request_body);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -367,7 +368,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_codex_fingerprint_convergence_to_decision(
|
||||
fn apply_provider_outbound_request_policies_to_decision(
|
||||
input: &LocalRequestedModelDecisionInput,
|
||||
decision: &mut AiExecutionDecision,
|
||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||
@@ -378,17 +379,17 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let Some(context) = input.codex_fingerprint_context.as_ref() else {
|
||||
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let applied = crate::ai_serving::transport::apply_codex_fingerprint_convergence_with_context(
|
||||
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||
transport,
|
||||
provider_api_format,
|
||||
context,
|
||||
&mut decision.provider_request_headers,
|
||||
provider_request_body,
|
||||
);
|
||||
if applied {
|
||||
if results.iter().any(|result| result.was_applied()) {
|
||||
decision.prompt_cache_key = provider_request_body
|
||||
.get("prompt_cache_key")
|
||||
.and_then(Value::as_str)
|
||||
@@ -396,6 +397,31 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
}
|
||||
if results.is_empty() {
|
||||
return;
|
||||
}
|
||||
for result in &results {
|
||||
tracing::debug!(
|
||||
event_name = "provider_outbound_policy_evaluated",
|
||||
log_type = "event",
|
||||
policy = ?result.policy,
|
||||
outcome = ?result.outcome,
|
||||
reason = ?result.reason,
|
||||
mutation_scope = ?result.mutation_scope,
|
||||
identity_scope = ?result.identity_scope,
|
||||
"provider outbound request policy evaluated"
|
||||
);
|
||||
}
|
||||
let Some(serde_json::Value::Object(report_context)) = decision.report_context.as_mut() else {
|
||||
return;
|
||||
};
|
||||
report_context.insert(
|
||||
"provider_outbound_policies".to_string(),
|
||||
serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"results": results,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
||||
@@ -484,7 +510,7 @@ pub(crate) fn build_local_requested_model_decision_input(
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -499,7 +525,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
body_json: &Value,
|
||||
client_api_format: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
input.codex_fingerprint_context =
|
||||
input.provider_outbound_context =
|
||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||
let selected_group = match state.routing_group_read_repository() {
|
||||
@@ -863,10 +889,32 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
||||
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||
},
|
||||
error => GatewayError::Client {
|
||||
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: error.to_string(),
|
||||
message: "requested routing group was not found".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not enabled".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not allowed for this principal".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_contract() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_headers() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -921,14 +969,9 @@ fn btree_headers_to_header_map(
|
||||
) -> Result<HeaderMap, GatewayError> {
|
||||
let mut output = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
||||
})?;
|
||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
||||
})?;
|
||||
let name = HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| invalid_routing_provider_headers())?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||
output.insert(name, value);
|
||||
}
|
||||
Ok(output)
|
||||
@@ -1151,6 +1194,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||
let secret = "private-group?token=Bearer-secret";
|
||||
|
||||
for error in [
|
||||
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||
] {
|
||||
let error = routing_selection_error(error);
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
ref message,
|
||||
} if !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||
let secret = "https://internal.example/?token=Bearer-secret";
|
||||
let contract_error = invalid_routing_provider_contract();
|
||||
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||
format!("Authorization: {secret}"),
|
||||
secret.to_string(),
|
||||
)]))
|
||||
.expect_err("invalid header should fail");
|
||||
|
||||
for (error, expected_message) in [
|
||||
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||
] {
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
ref message,
|
||||
} if message == expected_message && !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||
// AppState::new has no auth snapshot repository. Without the explicit
|
||||
@@ -1328,7 +1415,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1577,7 +1664,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1647,7 +1734,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -1716,6 +1803,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_codex_provider_outbound_policies_are_terminal_noop() {
|
||||
let mut input = sample_decision_input();
|
||||
input.routing_context = None;
|
||||
input.provider_outbound_context = Some(ProviderOutboundRequestContext::new(
|
||||
"logical-turn",
|
||||
1_700_000_000_123,
|
||||
));
|
||||
let mut decision = sample_codex_fingerprint_decision();
|
||||
decision.provider_type = Some("openai".to_string());
|
||||
decision.provider_api_format = Some("openai:responses".to_string());
|
||||
decision.client_api_format = Some("openai:responses".to_string());
|
||||
let mut transport = sample_codex_fingerprint_transport();
|
||||
transport.provider.provider_type = "openai".to_string();
|
||||
|
||||
let original_headers = decision.provider_request_headers.clone();
|
||||
let original_body = decision.provider_request_body.clone();
|
||||
apply_provider_request_routing_policy_to_decision(&input, &mut decision, Some(&transport))
|
||||
.expect("non-Codex terminal finalization should succeed");
|
||||
|
||||
assert_eq!(decision.provider_request_headers, original_headers);
|
||||
assert_eq!(decision.provider_request_body, original_body);
|
||||
assert!(decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
||||
let transport = sample_codex_fingerprint_transport();
|
||||
@@ -1732,8 +1848,8 @@ mod tests {
|
||||
});
|
||||
let mut with_mutation = sample_decision_input();
|
||||
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
||||
input.codex_fingerprint_context = Some(
|
||||
CodexFingerprintConvergenceContext::new(
|
||||
input.provider_outbound_context = Some(
|
||||
ProviderOutboundRequestContext::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
1_756_668_000_000,
|
||||
)
|
||||
@@ -1802,6 +1918,25 @@ mod tests {
|
||||
);
|
||||
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
||||
|
||||
let policy_results = decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.and_then(|policies| policies.get("results"))
|
||||
.and_then(Value::as_array)
|
||||
.expect("provider policy results");
|
||||
assert_eq!(
|
||||
policy_results.len(),
|
||||
1,
|
||||
"policy result count at {exit_name}"
|
||||
);
|
||||
assert_eq!(
|
||||
policy_results[0]["policy"],
|
||||
json!("codex_fingerprint_convergence")
|
||||
);
|
||||
assert_eq!(policy_results[0]["outcome"], json!("applied"));
|
||||
assert_eq!(policy_results[0]["reason"], json!("applied"));
|
||||
|
||||
let header_metadata: Value =
|
||||
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
||||
.expect("header turn metadata");
|
||||
|
||||
@@ -314,6 +314,20 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Same-format requests skip `apply_transport_request_body_semantics`, so the opt-in
|
||||
// Claude Code body mimicry has to be applied here as well.
|
||||
if crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
prepared.provider_api_format.as_str(),
|
||||
) {
|
||||
compatibility_edits.push(SameFormatProviderCompatibilityEdit {
|
||||
field: "body".to_string(),
|
||||
action: SameFormatProviderCompatibilityEditAction::ProviderCompatibilityRewrite,
|
||||
detail: "applied Claude Code body mimicry for provider compatibility".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let antigravity_auth = if prepared.is_antigravity {
|
||||
let mut antigravity_support = classify_local_antigravity_request_support(
|
||||
&transport,
|
||||
@@ -583,6 +597,11 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
source_model,
|
||||
codex_model_capabilities.as_ref(),
|
||||
);
|
||||
crate::ai_serving::transport::xai::insert_cli_identity_headers_if_needed(
|
||||
transport.as_ref(),
|
||||
prepared.provider_api_format.as_str(),
|
||||
&mut provider_request_headers,
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
redaction.redacted,
|
||||
|
||||
@@ -194,7 +194,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncA
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
@@ -246,7 +246,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt>
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
|
||||
@@ -76,10 +76,12 @@ pub(crate) fn build_local_execution_report_context(
|
||||
let RequestOrigin {
|
||||
client_ip,
|
||||
user_agent,
|
||||
forwarded_headers_trusted,
|
||||
} = parts
|
||||
.request_origin
|
||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
||||
let original_headers =
|
||||
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||
parts.original_request_body_json,
|
||||
parts.original_request_body_base64,
|
||||
@@ -106,15 +108,17 @@ pub(crate) fn build_local_execution_report_context(
|
||||
value,
|
||||
);
|
||||
}
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
if forwarded_headers_trusted {
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
}
|
||||
}
|
||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||
if let Some(policy) = parts.routing_policy {
|
||||
if let Ok(value) = serde_json::to_value(policy.execution_policy) {
|
||||
if let Ok(value) = serde_json::to_value(&policy.execution_policy) {
|
||||
extra_fields.insert(ROUTING_EXECUTION_POLICY_REPORT_FIELD.to_string(), value);
|
||||
}
|
||||
}
|
||||
@@ -189,6 +193,17 @@ pub(crate) fn build_local_execution_report_context(
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_report_context_original_headers(
|
||||
headers: &http::HeaderMap,
|
||||
forwarded_headers_trusted: bool,
|
||||
) -> BTreeMap<String, String> {
|
||||
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||
if !forwarded_headers_trusted {
|
||||
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||
}
|
||||
collected
|
||||
}
|
||||
|
||||
fn insert_request_path_fields(
|
||||
extra_fields: &mut Map<String, Value>,
|
||||
request_path: Option<&str>,
|
||||
@@ -258,8 +273,8 @@ mod tests {
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::{
|
||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
||||
LocalExecutionReportContextParts,
|
||||
build_local_execution_report_context, collect_report_context_original_headers,
|
||||
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||
};
|
||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||
use crate::ai_serving::RequestOrigin;
|
||||
@@ -289,6 +304,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||
|
||||
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||
assert_eq!(
|
||||
untrusted.get("user-agent").map(String::as_str),
|
||||
Some("test-client")
|
||||
);
|
||||
|
||||
let trusted = collect_report_context_original_headers(&headers, true);
|
||||
assert_eq!(
|
||||
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||
Some("spoofed-ja3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||
let auth_context = ExecutionRuntimeAuthContext {
|
||||
@@ -339,6 +374,7 @@ mod tests {
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: Some("203.0.113.8".to_string()),
|
||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||
forwarded_headers_trusted: false,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
@@ -491,7 +527,11 @@ mod tests {
|
||||
original_headers: &original_headers,
|
||||
request_path: None,
|
||||
request_query_string: None,
|
||||
request_origin: None,
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
forwarded_headers_trusted: true,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: None,
|
||||
|
||||
@@ -179,7 +179,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptS
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
@@ -224,7 +224,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAtte
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
|
||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
||||
GeminiFilesRequestBodyError,
|
||||
};
|
||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
use super::support::{
|
||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
if matches!(
|
||||
spec_metadata.decision_kind,
|
||||
GEMINI_FILES_GET_PLAN_KIND
|
||||
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||
{
|
||||
mark_skipped_local_gemini_files_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"gemini_file_mapping_mismatch",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(skip_reason) =
|
||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||
{
|
||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
file_name,
|
||||
})
|
||||
}
|
||||
|
||||
async fn candidate_matches_owned_gemini_file_mapping(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
input: &LocalGeminiFilesDecisionInput,
|
||||
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||
) -> bool {
|
||||
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||
return false;
|
||||
};
|
||||
let user_id = input.auth_context.user_id.trim();
|
||||
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(mapping)) = state
|
||||
.find_active_gemini_file_mapping_for_owner(
|
||||
file_name.as_str(),
|
||||
&attempt.eligible.transport.key.id,
|
||||
user_id,
|
||||
crate::clock::current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
|
||||
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||
if suffix.is_empty() || suffix.contains('/') {
|
||||
return None;
|
||||
}
|
||||
Some(format!("files/{suffix}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_gemini_file_name_from_path;
|
||||
|
||||
#[test]
|
||||
fn normalizes_supported_gemini_file_object_paths() {
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||
Some("files/abc-123".to_string())
|
||||
);
|
||||
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,7 +257,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptS
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
@@ -302,7 +302,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAtte
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::candidate_preparation::{
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::{
|
||||
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
||||
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
||||
@@ -16,8 +17,8 @@ use crate::ai_serving::transport::{
|
||||
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_special_headers, build_chatgpt_web_image_request_body,
|
||||
build_codex_openai_image_api_provider_request_body,
|
||||
apply_codex_openai_special_headers, apply_xai_upstream_payload_edits,
|
||||
build_chatgpt_web_image_request_body, build_codex_openai_image_api_provider_request_body,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
|
||||
default_model_for_openai_image_operation, normalize_openai_image_request,
|
||||
@@ -210,7 +211,7 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
upstream_is_stream,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body) = provider_request_body else {
|
||||
let Some(mut provider_request_body) = provider_request_body else {
|
||||
mark_skipped_local_openai_image_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
@@ -228,6 +229,11 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
.await;
|
||||
return None;
|
||||
};
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
);
|
||||
let Some(mut provider_request_headers) = (if is_grok {
|
||||
build_grok_browser_headers(GrokHeaderInput {
|
||||
transport,
|
||||
@@ -338,6 +344,25 @@ async fn resolve_local_openai_image_to_gemini_candidate_payload_parts(
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let provider_api_format = "gemini:generate_content";
|
||||
|
||||
// The gemini:generate_content URL hook rewrites an Antigravity endpoint to
|
||||
// /v1internal:, and this image path has no v1internal envelope to match it.
|
||||
// Skip the candidate instead of posting a bare Gemini body that upstream
|
||||
// would only reject.
|
||||
if is_antigravity_provider_transport(transport) {
|
||||
mark_skipped_local_openai_image_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
|
||||
@@ -109,7 +109,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptS
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::ai_serving::planner::{
|
||||
build_ai_execution_decision_response, resolve_transport_request_encoding_policy,
|
||||
AiExecutionDecisionResponseParts,
|
||||
};
|
||||
use crate::ai_serving::transport::xai::video::is_native_video_request;
|
||||
use crate::ai_serving::transport::{
|
||||
resolve_transport_execution_timeouts, resolve_transport_profile,
|
||||
};
|
||||
@@ -33,7 +34,7 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
let Some(resolved) = resolve_local_video_create_candidate_payload_parts(
|
||||
state, parts, body_json, trace_id, input, &attempt, spec,
|
||||
)
|
||||
.await
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -52,9 +53,32 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
.await;
|
||||
let transport_profile = resolve_transport_profile(&transport);
|
||||
let mut extra_fields = serde_json::Map::new();
|
||||
if is_native_video_request(&transport.provider.provider_type, parts.uri.path()) {
|
||||
extra_fields.insert(
|
||||
"video_client_protocol".to_string(),
|
||||
serde_json::json!("xai"),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(proxy_value) = build_request_trace_proxy_value(Some(&transport), proxy.as_ref()) {
|
||||
extra_fields.insert("proxy".to_string(), proxy_value);
|
||||
}
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai") {
|
||||
extra_fields.insert("video_provider_xai".into(), serde_json::json!(true));
|
||||
if let Some(duration) = resolved.provider_request_body.get("duration") {
|
||||
extra_fields.insert("video_duration".into(), duration.clone());
|
||||
}
|
||||
if parts.uri.path() == "/openai/v1/videos" {
|
||||
extra_fields.insert(
|
||||
"video_size".into(),
|
||||
body_json
|
||||
.get("size")
|
||||
.filter(|v| v.as_str().is_some_and(|s| !s.trim().is_empty()))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::json!("720x1280")),
|
||||
);
|
||||
}
|
||||
}
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||
auth_context: &input.auth_context,
|
||||
|
||||
@@ -3,15 +3,23 @@ use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::candidate_preparation::resolve_candidate_mapped_model;
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, resolve_candidate_mapped_model, OauthPreparationContext,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_video_create_spec_metadata;
|
||||
use crate::ai_serving::transport::xai::video::{
|
||||
convert_openai_video_request, is_explicit_native_video_path, is_native_video_request,
|
||||
};
|
||||
use crate::ai_serving::transport::{
|
||||
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
|
||||
resolve_video_create_auth, video_create_transport_unsupported_reason,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
};
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::AppState;
|
||||
use crate::ai_serving::{
|
||||
apply_xai_upstream_payload_edits, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
|
||||
PlannerAppState,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::support::{
|
||||
mark_skipped_local_video_candidate, mark_skipped_local_video_candidate_with_failure_diagnostic,
|
||||
@@ -37,11 +45,16 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
input: &LocalVideoCreateDecisionInput,
|
||||
attempt: &LocalVideoCreateCandidateAttempt,
|
||||
spec: LocalVideoCreateSpec,
|
||||
) -> Option<LocalVideoCreateCandidatePayloadParts> {
|
||||
) -> Result<Option<LocalVideoCreateCandidatePayloadParts>, GatewayError> {
|
||||
let spec_metadata = local_video_create_spec_metadata(spec);
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
if is_explicit_native_video_path(parts.uri.path())
|
||||
&& !transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let provider_family = provider_video_create_family(spec.family);
|
||||
let transport_unsupported_reason = video_create_transport_unsupported_reason(
|
||||
@@ -60,23 +73,39 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let auth = resolve_video_create_auth(transport, provider_family);
|
||||
let Some((auth_header, auth_value)) = auth else {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
PlannerAppState::new(state),
|
||||
transport,
|
||||
candidate,
|
||||
resolve_video_create_auth(transport, provider_family),
|
||||
OauthPreparationContext {
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_auth_unavailable",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
api_format: spec_metadata.api_format,
|
||||
operation: "video_create_candidate_request",
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(skip_reason) => {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let auth_header = prepared_candidate.auth_header;
|
||||
let auth_value = prepared_candidate.auth_value;
|
||||
|
||||
let mapped_model = match resolve_candidate_mapped_model(candidate) {
|
||||
Ok(mapped_model) => mapped_model,
|
||||
@@ -91,7 +120,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -117,10 +146,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(provider_request_body) = build_video_create_request_body(
|
||||
let Some(mut provider_request_body) = build_video_create_request_body(
|
||||
body_json,
|
||||
provider_family,
|
||||
&mapped_model,
|
||||
@@ -142,11 +171,28 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
&& !is_native_video_request(&transport.provider.provider_type, parts.uri.path())
|
||||
{
|
||||
provider_request_body =
|
||||
convert_openai_video_request(&provider_request_body).map_err(|message| {
|
||||
GatewayError::Client {
|
||||
status: http::StatusCode::BAD_REQUEST,
|
||||
message: message.to_string(),
|
||||
}
|
||||
})?;
|
||||
}
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
spec_metadata.api_format,
|
||||
);
|
||||
|
||||
let Some(provider_request_headers) =
|
||||
build_video_create_headers(ProviderVideoCreateHeadersInput {
|
||||
transport,
|
||||
headers: effective_headers,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
@@ -170,10 +216,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Some(LocalVideoCreateCandidatePayloadParts {
|
||||
Ok(Some(LocalVideoCreateCandidatePayloadParts {
|
||||
transport: Arc::clone(transport),
|
||||
auth_header,
|
||||
auth_value,
|
||||
@@ -181,7 +227,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
provider_request_headers,
|
||||
provider_request_body,
|
||||
upstream_url,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn provider_video_create_family(family: LocalVideoCreateFamily) -> ProviderVideoCreateFamily {
|
||||
|
||||
@@ -504,10 +504,13 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
@@ -614,10 +617,13 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
||||
Some(&"acc-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
for name in ["x-client-request-id", "session-id", "thread-id"] {
|
||||
@@ -698,10 +704,13 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
||||
Some(&"kept-by-rule-request".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert_eq!(
|
||||
headers
|
||||
.keys()
|
||||
@@ -762,10 +771,13 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,39 +1,21 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub(crate) fn is_deepseek_provider(provider_type: &str, base_url: &str) -> bool {
|
||||
let provider_type = provider_type.trim().to_ascii_lowercase();
|
||||
if matches!(
|
||||
provider_type.as_str(),
|
||||
"deepseek" | "deepseek_openai" | "deepseek_anthropic" | "deepseek_compatible"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
pub(crate) fn is_deepseek_provider(_provider_type: &str, base_url: &str) -> bool {
|
||||
let Some(host) = base_url_host(base_url) else {
|
||||
return false;
|
||||
};
|
||||
host == "deepseek.com" || host.ends_with(".deepseek.com")
|
||||
}
|
||||
|
||||
fn is_deepseek_model(provider_model: &str) -> bool {
|
||||
let provider_model = provider_model.trim().to_ascii_lowercase();
|
||||
let leaf = provider_model
|
||||
.rsplit(['/', ':'])
|
||||
.next()
|
||||
.unwrap_or(provider_model.as_str());
|
||||
leaf == "deepseek" || leaf.starts_with("deepseek-") || leaf.starts_with("deepseek_")
|
||||
}
|
||||
|
||||
fn is_deepseek_upstream(provider_type: &str, base_url: &str, provider_model: &str) -> bool {
|
||||
is_deepseek_provider(provider_type, base_url) || is_deepseek_model(provider_model)
|
||||
// 仅官方接口启用专用兼容;供应商类型和模型名称不能代表第三方接口的行为。
|
||||
host == "api.deepseek.com"
|
||||
}
|
||||
|
||||
pub(crate) fn openai_responses_reasoning_replay_policy(
|
||||
provider_type: &str,
|
||||
base_url: &str,
|
||||
provider_model: &str,
|
||||
_provider_model: &str,
|
||||
) -> crate::ai_serving::OpenAiResponsesReasoningReplayPolicy {
|
||||
if is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if provider_type.trim().eq_ignore_ascii_case("xai") {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
} else if is_deepseek_provider(provider_type, base_url) {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
} else {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
@@ -47,11 +29,7 @@ pub(crate) fn apply_deepseek_tool_call_thinking_compat(
|
||||
provider_api_format: &str,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if !is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if !is_deepseek_provider(provider_type, base_url) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -155,6 +133,9 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
provider_request_body: &mut Value,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
// 携带 tools 时,所有历史 reasoning_content 都须完整回传,包括未调用工具的轮次。
|
||||
// 无 tools 时允许回传,且 prefix 续写需要保留输入;因此原样保留 messages,
|
||||
// 不删除思考内容,也不以空字符串冒充缺失内容,由上游校验请求是否完整。
|
||||
let disabled = source_disables_thinking(original_request_body, provider_request_body);
|
||||
set_deepseek_thinking_type(
|
||||
provider_request_body,
|
||||
@@ -170,33 +151,6 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
{
|
||||
object.remove("reasoning_effort");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(messages) = object.get_mut("messages").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
for message in messages {
|
||||
let Some(message_object) = message.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
let is_assistant = message_object
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"));
|
||||
if !is_assistant {
|
||||
continue;
|
||||
}
|
||||
if message_object
|
||||
.get("reasoning_content")
|
||||
.is_some_and(|value| !value.is_null())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
message_object.insert(
|
||||
"reasoning_content".to_string(),
|
||||
Value::String(String::new()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,8 +241,29 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_by_type_or_host() {
|
||||
assert!(is_deepseek_provider(
|
||||
fn xai_reasoning_policy_comes_from_provider_type() {
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"xai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"openai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_only_by_official_host() {
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://relay.example.com"
|
||||
));
|
||||
@@ -298,6 +273,16 @@ mod tests {
|
||||
));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com:443/v1"));
|
||||
assert!(!is_deepseek_provider("custom", "https://deepseek.com"));
|
||||
assert!(!is_deepseek_provider("custom", "deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider(
|
||||
"custom",
|
||||
" HTTPS://API.DEEPSEEK.COM:443/beta "
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://other.deepseek.com/v1"
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"custom",
|
||||
"https://example.com/deepseek"
|
||||
@@ -319,6 +304,15 @@ mod tests {
|
||||
"https://evil.example/[email protected]"
|
||||
));
|
||||
assert!(!is_deepseek_provider("custom", "ftp://api.deepseek.com/v1"));
|
||||
assert!(!is_deepseek_provider("deepseek", ""));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"deepseek",
|
||||
"https://deepseek.com/v1",
|
||||
"deepseek-chat",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"custom",
|
||||
@@ -341,7 +335,7 @@ mod tests {
|
||||
"https://api.b.ai/v1",
|
||||
"deepseek-v4-flash",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
@@ -353,6 +347,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn official_deepseek_host_enables_compat_without_type_or_model_hints() {
|
||||
for base_url in [
|
||||
"https://api.deepseek.com/v1",
|
||||
"https://api.deepseek.com/beta",
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "mapped-model",
|
||||
"messages": [{"role": "assistant", "content": "answer"}]
|
||||
});
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"custom",
|
||||
base_url,
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy("custom", base_url, "mapped-model"),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_deepseek_host_preserves_production_shaped_opaque_reasoning_replay() {
|
||||
let reasoning_items = (0..66)
|
||||
@@ -444,7 +465,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_adds_thinking_and_empty_reasoning_content() {
|
||||
fn openai_chat_deepseek_enables_thinking_without_fabricating_reasoning() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
@@ -467,11 +488,11 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
assert!(body["messages"][1].get("reasoning_content").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_relay_deepseek_model_adds_chat_thinking_compat() {
|
||||
fn custom_relay_deepseek_model_preserves_chat_request() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
@@ -484,6 +505,7 @@ mod tests {
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
||||
]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
@@ -493,8 +515,211 @@ mod tests {
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn third_party_hosts_ignore_deepseek_type_and_model_hints() {
|
||||
for provider_type in [
|
||||
"custom",
|
||||
"deepseek",
|
||||
"deepseek_openai",
|
||||
"deepseek_anthropic",
|
||||
"deepseek_compatible",
|
||||
] {
|
||||
for provider_model in [
|
||||
"other-model",
|
||||
"deepseek-chat",
|
||||
"deepseek-reasoner",
|
||||
"deepseek-v3",
|
||||
"deepseek-v4-flash",
|
||||
"vendor/deepseek-chat",
|
||||
"vendor:deepseek-reasoner",
|
||||
] {
|
||||
let base_url = "https://relay.example.com/v1";
|
||||
assert!(!is_deepseek_provider(provider_type, base_url));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
provider_type,
|
||||
base_url,
|
||||
provider_model
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
|
||||
for api_format in ["openai:chat", "claude:messages"] {
|
||||
let original = json!({
|
||||
"model": provider_model,
|
||||
"messages": [{
|
||||
"role": "assistant",
|
||||
"content": "answer",
|
||||
"reasoning_content": "original plan"
|
||||
}]
|
||||
});
|
||||
let mut body = original.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
provider_type,
|
||||
base_url,
|
||||
api_format,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
body, original,
|
||||
"{provider_type} / {provider_model} / {api_format}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_history_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Compare 9.11 and 9.8"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "9.8 is greater",
|
||||
"reasoning_content": "Compare the decimal places.\n9.80 > 9.11."
|
||||
},
|
||||
{"role": "user", "content": "Explain again"},
|
||||
{"role": "assistant", "content": "Compare 9.80 with 9.11"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_across_all_tool_turns() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}],
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"reasoning_content": "Check the weather before answering.\nKeep this full plan.",
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"}
|
||||
}]
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Cloudy"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "It is cloudy",
|
||||
"reasoning_content": "The weather result is available; summarize it."
|
||||
},
|
||||
{"role": "user", "content": "Should I take an umbrella?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "An umbrella may be useful",
|
||||
"reasoning_content": "Use the previous weather result without another tool call."
|
||||
},
|
||||
{"role": "user", "content": "Why?"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
let tools = body["tools"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["tools"], tools);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_does_not_fabricate_missing_tool_reasoning() {
|
||||
for tools in [
|
||||
json!([]),
|
||||
json!([{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}]),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": tools,
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "missing"},
|
||||
{"role": "assistant", "content": "null", "reasoning_content": null},
|
||||
{"role": "assistant", "content": "empty", "reasoning_content": ""},
|
||||
{"role": "assistant", "content": "answer", "reasoning_content": "original plan"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_prefix_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 1 + 1?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"prefix": true,
|
||||
"content": "",
|
||||
"reasoning_content": "Start by adding one to one."
|
||||
}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/beta",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -182,7 +182,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSour
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
@@ -232,7 +232,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttempt
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
|
||||
@@ -4,6 +4,10 @@ use std::sync::Arc;
|
||||
use aether_contracts::ResolvedTransportProfile;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::antigravity::{
|
||||
build_antigravity_v1internal_provider_request, AntigravityV1InternalRequestError,
|
||||
AntigravityV1InternalRequestInput, ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME,
|
||||
};
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
||||
OauthPreparationContext,
|
||||
@@ -26,6 +30,7 @@ use crate::ai_serving::planner::standard::{
|
||||
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
||||
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
||||
};
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::kiro::{
|
||||
build_kiro_provider_headers, build_kiro_provider_request_body,
|
||||
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
||||
@@ -587,7 +592,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -837,6 +842,29 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_antigravity_provider_transport(transport)
|
||||
{
|
||||
return Ok(build_antigravity_cross_format_payload_parts(
|
||||
state,
|
||||
parts,
|
||||
trace_id,
|
||||
body_json,
|
||||
input,
|
||||
attempt,
|
||||
transport,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
prepared_candidate.mapped_model,
|
||||
prepared_candidate.auth_header,
|
||||
prepared_candidate.auth_value,
|
||||
provider_request_body,
|
||||
upstream_is_stream,
|
||||
redaction.redacted,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_gemini_cli_provider_transport(transport)
|
||||
{
|
||||
@@ -963,6 +991,145 @@ fn apply_transport_request_body_semantics(
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_antigravity_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
trace_id: &str,
|
||||
original_body_json: &serde_json::Value,
|
||||
input: &LocalStandardDecisionInput,
|
||||
attempt: &LocalStandardCandidateAttempt,
|
||||
transport: &Arc<GatewayProviderTransportSnapshot>,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
mapped_model: String,
|
||||
auth_header: String,
|
||||
auth_value: String,
|
||||
gemini_request_body: Value,
|
||||
upstream_is_stream: bool,
|
||||
request_redacted: bool,
|
||||
) -> Option<LocalStandardCandidatePayloadParts> {
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let resolved =
|
||||
match build_antigravity_v1internal_provider_request(AntigravityV1InternalRequestInput {
|
||||
state,
|
||||
parts,
|
||||
transport,
|
||||
trace_id,
|
||||
mapped_model: &mapped_model,
|
||||
provider_api_format,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
request_headers: effective_headers,
|
||||
original_request_body: original_body_json,
|
||||
gemini_request_body: &gemini_request_body,
|
||||
upstream_is_stream,
|
||||
same_format: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(resolved) => resolved,
|
||||
Err(AntigravityV1InternalRequestError::TransportUnsupported) => {
|
||||
mark_skipped_local_standard_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::EnvelopeUnsupported) => {
|
||||
mark_skipped_local_standard_candidate_with_extra_data(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"provider_request_body_build_failed",
|
||||
request_body_build_failure_extra_data(
|
||||
original_body_json,
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::UpstreamUrlUnavailable) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"upstream_url_missing",
|
||||
CandidateFailureDiagnostic::upstream_url_missing(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_url",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::HeaderRulesApplyFailed) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_header_rules_apply_failed",
|
||||
CandidateFailureDiagnostic::header_rules_apply_failed(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_headers",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let mut provider_request_headers = resolved.headers.headers;
|
||||
apply_codex_openai_special_headers(
|
||||
&mut provider_request_headers,
|
||||
&resolved.body,
|
||||
effective_headers,
|
||||
resolved.transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
Some(trace_id),
|
||||
resolved.transport.key.decrypted_auth_config.as_deref(),
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
request_redacted,
|
||||
);
|
||||
|
||||
Some(LocalStandardCandidatePayloadParts {
|
||||
auth_header: resolved.headers.auth_header,
|
||||
auth_value: resolved.headers.auth_value,
|
||||
mapped_model,
|
||||
provider_api_format: provider_api_format.to_string(),
|
||||
provider_request_body: resolved.body,
|
||||
provider_request_headers,
|
||||
upstream_url: resolved.upstream_url,
|
||||
upstream_is_stream,
|
||||
envelope_name: Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME),
|
||||
transport: resolved.transport,
|
||||
transport_profile: None,
|
||||
request_redacted,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_gemini_cli_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::ai_serving::transport::apply_standard_provider_request_body_rules_wit
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_responses_chat_body_edits,
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives as surface_build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context as surface_build_cross_format_openai_chat_request_body,
|
||||
build_local_openai_chat_request_body_with_model_directives as surface_build_local_openai_chat_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -73,9 +73,11 @@ pub(crate) fn build_cross_format_openai_chat_request_body(
|
||||
let provider_request_body = surface_build_cross_format_openai_chat_request_body(
|
||||
body_json,
|
||||
mapped_model,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
user_api_key_id,
|
||||
)?;
|
||||
let mut provider_request_body =
|
||||
apply_standard_provider_request_body_rules_with_request_headers(
|
||||
@@ -110,6 +112,42 @@ pub(crate) fn build_cross_format_openai_chat_request_body(
|
||||
Some(provider_request_body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_chat_route_preserves_tool_schema_and_alternate_responses_shape() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let body = json!({"model": "client", "messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{"type": "function", "function": {"name": "probe", "parameters": schema}}]});
|
||||
let responses_body = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for input in [body, responses_body] {
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output = build_cross_format_openai_chat_request_body(
|
||||
&input,
|
||||
"claude-test",
|
||||
provider,
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_cross_format_openai_chat_upstream_url(
|
||||
parts: &http::request::Parts,
|
||||
transport: &GatewayProviderTransportSnapshot,
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
||||
use crate::ai_serving::transport::apply_standard_provider_request_body_rules_with_request_headers;
|
||||
use crate::ai_serving::{
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope as surface_build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context as surface_build_cross_format_openai_responses_request_body,
|
||||
build_local_openai_responses_request_body_with_model_directives as surface_build_local_openai_responses_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -218,6 +218,7 @@ pub(crate) fn build_cross_format_openai_responses_request_body_with_codex_model_
|
||||
body_json,
|
||||
mapped_model,
|
||||
client_api_format,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
@@ -274,6 +275,41 @@ pub(crate) fn build_local_openai_responses_upstream_url(
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_responses_route_preserves_tool_schema_without_changing_public_gemini() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let input = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output =
|
||||
build_cross_format_openai_responses_request_body_with_codex_model_capabilities(
|
||||
&input,
|
||||
"claude-test",
|
||||
"openai:responses",
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
provider,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
Some("antigravity-schema-test"),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_cross_format_openai_responses_upstream_url(
|
||||
parts: &http::request::Parts,
|
||||
transport: &GatewayProviderTransportSnapshot,
|
||||
|
||||
+16
-6
@@ -140,7 +140,7 @@ fn finalize_openai_chat_provider_request_body(
|
||||
mapped_model,
|
||||
source_model,
|
||||
);
|
||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
let finalization_failure = crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
provider_request_body,
|
||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||
source_api_format: "openai:chat",
|
||||
@@ -170,7 +170,17 @@ fn finalize_openai_chat_provider_request_body(
|
||||
provider_api_format,
|
||||
"openai_chat_request_finalization",
|
||||
)
|
||||
})
|
||||
});
|
||||
if finalization_failure.is_none() {
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Chat -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
provider_request_body,
|
||||
transport,
|
||||
provider_api_format,
|
||||
);
|
||||
}
|
||||
finalization_failure
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2183,7 +2193,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -2741,7 +2751,7 @@ mod tests {
|
||||
.provider_request_headers
|
||||
.get("x-client-version")
|
||||
.map(String::as_str),
|
||||
Some("1.2.3")
|
||||
Some("4.3.0")
|
||||
);
|
||||
assert_eq!(
|
||||
payload
|
||||
@@ -2761,9 +2771,9 @@ mod tests {
|
||||
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
||||
assert_eq!(
|
||||
payload.provider_request_body["userAgent"],
|
||||
"antigravity/cli/1.0.16 (aidev_client; os_type=linux; arch=arm64; auth_method=consumer)"
|
||||
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||
);
|
||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
||||
assert!(payload.provider_request_body.get("requestType").is_none());
|
||||
assert!(payload.provider_request_body.get("contents").is_none());
|
||||
assert!(payload.provider_request_body["request"]
|
||||
.get("contents")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode};
|
||||
use async_trait::async_trait;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
@@ -124,7 +124,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttem
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
@@ -207,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
|
||||
async fn next_raw_attempt_with_target_select(
|
||||
&mut self,
|
||||
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
||||
let select_window = openai_chat_stream_target_select_window();
|
||||
let select_window = openai_chat_stream_target_select_window_for_mode(
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.scheduling_mode),
|
||||
);
|
||||
if select_window <= 1 {
|
||||
return self.next_raw_attempt_linear().await;
|
||||
}
|
||||
@@ -365,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize {
|
||||
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
||||
}
|
||||
|
||||
fn openai_chat_stream_target_select_window_for_mode(
|
||||
scheduling_mode: Option<RoutingSchedulingMode>,
|
||||
) -> usize {
|
||||
if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) {
|
||||
return 1;
|
||||
}
|
||||
openai_chat_stream_target_select_window()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TargetSelectCandidateIdentity<'a> {
|
||||
provider_id: &'a str,
|
||||
@@ -574,4 +588,14 @@ mod tests {
|
||||
|
||||
assert_eq!(select_target_index(19, &choices), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_order_disables_stream_target_selection() {
|
||||
assert_eq!(
|
||||
openai_chat_stream_target_select_window_for_mode(Some(
|
||||
RoutingSchedulingMode::FixedOrder,
|
||||
)),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSo
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
|
||||
+12
-27
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiStreamAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_stream_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = effective_upstream_is_stream,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiSyncAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_sync_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = payload.upstream_is_stream,
|
||||
|
||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
||||
|
||||
use crate::ai_serving::build_request_trace_proxy_value;
|
||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::planner::report_context::{
|
||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
&resolved.transport,
|
||||
);
|
||||
|
||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
||||
debug!(
|
||||
event_name = "local_openai_responses_decision_payload_built",
|
||||
log_type = "debug",
|
||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
client_api_format = spec_metadata.api_format,
|
||||
provider_api_format = %resolved.provider_api_format,
|
||||
request_path = %parts.uri.path(),
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||
request_query = ?log_request_query,
|
||||
upstream_base_url = %log_base_url,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream = resolved.upstream_is_stream,
|
||||
has_envelope = resolved.envelope_name.is_some(),
|
||||
"gateway built local openai responses decision payload"
|
||||
|
||||
+14
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
||||
};
|
||||
use crate::ai_serving::planner::redaction::{
|
||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||
sanitize_upstream_url_for_log,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||
use crate::ai_serving::planner::standard::{
|
||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -636,6 +635,13 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
{
|
||||
log_responses_to_chat_tool_conversion(trace_id, body_json, &base_provider_request_body);
|
||||
}
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Responses -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
provider_api_format,
|
||||
);
|
||||
let provider_request_body = base_provider_request_body;
|
||||
|
||||
if let Some(kiro_auth) = kiro_auth.as_ref() {
|
||||
@@ -867,17 +873,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
||||
let log_custom_path = transport
|
||||
.endpoint
|
||||
.custom_path
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_upstream_url_resolved",
|
||||
@@ -893,12 +892,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
base_url = %log_base_url,
|
||||
custom_path = ?log_custom_path,
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
request_path = %parts.uri.path(),
|
||||
request_query = ?log_request_query,
|
||||
mapped_model = %mapped_model,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses upstream url"
|
||||
);
|
||||
@@ -2011,8 +2012,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
};
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||
log_type = "debug",
|
||||
@@ -2027,7 +2026,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses kiro upstream url"
|
||||
);
|
||||
|
||||
@@ -488,6 +488,7 @@ impl ResponsesWebSocketBodyNormalization {
|
||||
digest.update([match self.reasoning_replay_policy {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds => 0,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque => 1,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted => 2,
|
||||
}]);
|
||||
update_normalization_optional_json_digest(&mut digest, self.model_directive_patch.as_ref());
|
||||
digest.finalize().into()
|
||||
@@ -729,117 +730,6 @@ fn update_normalization_codex_capabilities_digest(
|
||||
update_normalization_string_vec_digest(digest, &capabilities.supported_service_tiers);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod continuation_fingerprint_tests {
|
||||
use http::HeaderValue;
|
||||
use serde_json::json;
|
||||
|
||||
use super::ResponsesWebSocketBodyNormalization;
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
||||
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
||||
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_changes_with_effective_contract() {
|
||||
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
||||
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
||||
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
||||
);
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_policy.continuation_fingerprint()
|
||||
);
|
||||
|
||||
let changed_patch = base
|
||||
.clone()
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_patch.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "x-contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
||||
second
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
||||
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "X-Contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules.clone());
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("disabled"));
|
||||
|
||||
assert_ne!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"a header that controls an effective body-rule condition remains part of the contract"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds one upstream decision for a Responses WebSocket turn. The session
|
||||
/// reuses this decision for same-model turns and invokes the planner again when
|
||||
/// a later `response.create` changes the public model.
|
||||
@@ -1058,3 +948,114 @@ async fn release_responses_websocket_planning_lease(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod continuation_fingerprint_tests {
|
||||
use http::HeaderValue;
|
||||
use serde_json::json;
|
||||
|
||||
use super::ResponsesWebSocketBodyNormalization;
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
||||
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
||||
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_changes_with_effective_contract() {
|
||||
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
||||
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
||||
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
||||
);
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_policy.continuation_fingerprint()
|
||||
);
|
||||
|
||||
let changed_patch = base
|
||||
.clone()
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_patch.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "x-contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
||||
second
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
||||
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "X-Contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules.clone());
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("disabled"));
|
||||
|
||||
assert_ne!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"a header that controls an effective body-rule condition remains part of the contract"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,7 +166,7 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAtte
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
@@ -216,7 +216,7 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStream
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy)
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::constants::{
|
||||
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
|
||||
};
|
||||
@@ -97,11 +95,13 @@ impl<'a> PlannerAppState<'a> {
|
||||
),
|
||||
GatewayError,
|
||||
> {
|
||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||
let wait_deadline = Instant::now() + wait_timeout;
|
||||
let mut attempt_now_unix_secs = now_unix_secs;
|
||||
loop {
|
||||
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||
self.app(),
|
||||
auth_snapshot,
|
||||
now_unix_secs,
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||
|attempt_now_unix_secs| async move {
|
||||
let result = crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons_for_request_operation(
|
||||
self.app().data.as_ref(),
|
||||
self.app(),
|
||||
@@ -118,21 +118,13 @@ impl<'a> PlannerAppState<'a> {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
||||
let auth_limit_blocked = crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
||||
&result.0, &result.1,
|
||||
) {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= wait_deadline {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let remaining = wait_deadline.duration_since(now);
|
||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
||||
attempt_now_unix_secs = current_unix_secs();
|
||||
}
|
||||
);
|
||||
Ok((result, auth_limit_blocked))
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -178,13 +170,14 @@ impl<'a> PlannerAppState<'a> {
|
||||
now_unix_secs: u64,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||
let wait_deadline = Instant::now() + wait_timeout;
|
||||
let mut attempt_now_unix_secs = now_unix_secs;
|
||||
|
||||
loop {
|
||||
let (result, auth_limit_blocked) = crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
||||
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||
self.app(),
|
||||
auth_snapshot,
|
||||
now_unix_secs,
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||
|attempt_now_unix_secs| {
|
||||
crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
||||
self.app().data.as_ref(),
|
||||
self.app(),
|
||||
candidate_api_format,
|
||||
@@ -195,20 +188,8 @@ impl<'a> PlannerAppState<'a> {
|
||||
attempt_now_unix_secs,
|
||||
ordering_config,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !auth_limit_blocked {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= wait_deadline {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let remaining = wait_deadline.duration_since(now);
|
||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
||||
attempt_now_unix_secs = current_unix_secs();
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,13 +12,16 @@ pub(crate) use aether_ai_formats::api::{
|
||||
apply_codex_openai_responses_websocket_continuation_body_edits_with_source_model_and_capabilities,
|
||||
apply_codex_openai_special_headers, apply_model_directive_mapping_patch,
|
||||
apply_model_directive_overrides_from_model, apply_model_directive_overrides_from_request,
|
||||
apply_openai_responses_compact_special_body_edits, build_chatgpt_web_image_request_body,
|
||||
apply_openai_responses_compact_special_body_edits, apply_xai_upstream_payload_edits,
|
||||
apply_xai_upstream_payload_edits_with_client, build_chatgpt_web_image_request_body,
|
||||
build_codex_model_catalog_metadata, build_codex_openai_image_api_provider_request_body,
|
||||
build_core_error_body_for_client_format, build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context,
|
||||
build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_gemini_image_response_from_openai_image_response,
|
||||
build_gemini_image_response_from_openai_responses_image_response, build_generated_tool_call_id,
|
||||
@@ -177,10 +180,11 @@ pub(crate) use aether_ai_formats::{
|
||||
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
||||
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
||||
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
||||
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
|
||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||
strip_incompatible_openai_responses_reasoning_items,
|
||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||
CODEX_CLIENT_VERSION, OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
};
|
||||
|
||||
pub(crate) fn plan_kind_matches_api_operation(
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||
};
|
||||
use aether_runtime_state::RuntimeState;
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::GatewayError;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||
|
||||
pub(crate) async fn hydrate_openai_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
request: &Value,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||
let runtime_state = state.runtime_state();
|
||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||
warn!(
|
||||
event_name = "openai_response_history_read_failed",
|
||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
let Some(payload) = payload else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&payload,
|
||||
) else {
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
event_name = "openai_response_history_decryption_failed",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway rejected undecryptable shared OpenAI response history"
|
||||
);
|
||||
return Err(GatewayError::Internal(
|
||||
"OpenAI response history decryption failed".to_string(),
|
||||
));
|
||||
};
|
||||
if let Err(error) =
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||
{
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_response_history_record(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
record: ResponseHistoryRecord,
|
||||
) {
|
||||
let runtime_state = state.runtime_state();
|
||||
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&record.payload,
|
||||
) else {
|
||||
warn!(
|
||||
event_name = "openai_response_history_encryption_unavailable",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway refused to persist unencrypted OpenAI response history"
|
||||
);
|
||||
return;
|
||||
};
|
||||
if let Err(error) = runtime_state
|
||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
||||
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_converted_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
report_context: &Value,
|
||||
response: Option<&Value>,
|
||||
) {
|
||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
||||
return;
|
||||
};
|
||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||
persist_response_history_record(runtime_state, record).await;
|
||||
persist_response_history_record(state, record).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||
};
|
||||
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||
|
||||
fn response_history_test_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("test state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||
MemoryRuntimeStateConfig::default(),
|
||||
)))
|
||||
}
|
||||
|
||||
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||
let expires_at_unix_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(3600);
|
||||
json!({
|
||||
"version": 1,
|
||||
"response_id": response_id,
|
||||
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||
"expires_at_unix_secs": expires_at_unix_secs,
|
||||
"transcript": [{"type": "message", "content": marker}],
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_encrypted_history_v1";
|
||||
let scope = "response-history-encrypted-scope";
|
||||
let marker = "private-response-history-marker";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, marker);
|
||||
|
||||
persist_response_history_record(
|
||||
&state,
|
||||
ResponseHistoryRecord {
|
||||
storage_key: storage_key.clone(),
|
||||
payload,
|
||||
ttl: Duration::from_secs(6 * 60 * 60),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stored = state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.expect("history should be persisted");
|
||||
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||
&stored
|
||||
));
|
||||
assert!(!stored.contains(marker));
|
||||
|
||||
hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
.expect("encrypted history should hydrate");
|
||||
assert!(crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_legacy_history_v1";
|
||||
let scope = "response-history-legacy-scope";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||
state
|
||||
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||
.await
|
||||
.expect("legacy history should store");
|
||||
|
||||
let result = hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
assert!(state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,11 +58,16 @@ pub(crate) mod windsurf {
|
||||
pub(crate) use aether_provider_transport::windsurf::*;
|
||||
}
|
||||
|
||||
pub(crate) mod xai {
|
||||
pub(crate) use aether_provider_transport::xai::*;
|
||||
}
|
||||
|
||||
pub(crate) use aether_provider_transport::{
|
||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_standard_provider_request_body_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_provider_outbound_request_policies,
|
||||
apply_standard_provider_request_body_rules,
|
||||
apply_standard_provider_request_body_rules_with_request_headers,
|
||||
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
||||
body_rules_handle_path, body_rules_have_enabled_rules,
|
||||
@@ -112,7 +117,11 @@ pub(crate) use aether_provider_transport::{
|
||||
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
||||
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
||||
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
||||
ProviderOpenAiImageHeadersInput, ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
ProviderOpenAiImageHeadersInput, ProviderOutboundRequestContext,
|
||||
ProviderOutboundRequestIdentityScope, ProviderOutboundRequestMutationScope,
|
||||
ProviderOutboundRequestPolicy, ProviderOutboundRequestPolicyOutcome,
|
||||
ProviderOutboundRequestPolicyReason, ProviderOutboundRequestPolicyResult,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
||||
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
||||
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
||||
@@ -121,5 +130,5 @@ pub(crate) use aether_provider_transport::{
|
||||
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
||||
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
||||
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
||||
GROK_RATE_LIMITS_PATH, WINDSURF_ENVELOPE_NAME,
|
||||
GROK_RATE_LIMITS_PATH, PROVIDER_OUTBOUND_CONTEXT_MAX_VALUE_BYTES, WINDSURF_ENVELOPE_NAME,
|
||||
};
|
||||
|
||||
@@ -53,6 +53,8 @@ const AI_ANY_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/operations/{*operation_path}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{*video_path}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{*video_path}",
|
||||
"/upload/v1beta/files",
|
||||
"/v1beta/files",
|
||||
"/v1beta/files/{*file_path}",
|
||||
|
||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
||||
"internal_gateway": {
|
||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||
"status": "rust_native_control_plane",
|
||||
"status": state.internal_gateway_auth_status(),
|
||||
},
|
||||
},
|
||||
"features": {
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Request, State};
|
||||
use axum::http::{self, HeaderValue, StatusCode};
|
||||
use axum::middleware::{self, Next};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use axum::{Json, Router};
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::async_task::{
|
||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
||||
use crate::router::metrics;
|
||||
use crate::state::AppState;
|
||||
|
||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
||||
router
|
||||
#[derive(Clone, Copy)]
|
||||
struct OperationalPermission {
|
||||
required_permissions: &'static [&'static str],
|
||||
write: bool,
|
||||
requires_full_admin_role: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn mount_operational_routes(
|
||||
router: Router<AppState>,
|
||||
state: AppState,
|
||||
) -> Router<AppState> {
|
||||
let operational = Router::<AppState>::new()
|
||||
.route("/_gateway/metrics", get(metrics))
|
||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||
.route(
|
||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
||||
"/_gateway/audit/request-usage/{request_id}",
|
||||
get(get_request_usage_audit),
|
||||
)
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
state,
|
||||
authorize_operational_request,
|
||||
));
|
||||
router.merge(operational)
|
||||
}
|
||||
|
||||
async fn authorize_operational_request(
|
||||
State(state): State<AppState>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response<Body> {
|
||||
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||
return operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational route permission is not configured",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let Some(remote_addr) = request
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|value| value.0)
|
||||
else {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let headers = request.headers().clone();
|
||||
let uri = request.uri().clone();
|
||||
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||
return operational_auth_required_response();
|
||||
}
|
||||
|
||||
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||
Ok(Some(principal)) => {
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||
&state, &headers, client_ip,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(authenticated) => authenticated,
|
||||
Err(
|
||||
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||
) => return operational_auth_required_response(),
|
||||
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
let missing_permission =
|
||||
permission
|
||||
.required_permissions
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|required| {
|
||||
!management_token_has_operational_permission(
|
||||
&authenticated.permissions,
|
||||
required,
|
||||
)
|
||||
});
|
||||
if let Some(required_permission) = missing_permission {
|
||||
return operational_permission_denied_response(required_permission);
|
||||
}
|
||||
|
||||
let client_ip = client_ip.to_string();
|
||||
if let Err(err) = state
|
||||
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
token_id = %authenticated.token.id,
|
||||
error = ?err,
|
||||
"gateway failed to record operational management token usage"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut response = next.run(request).await;
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||
if path == "/_gateway/metrics" {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: if write {
|
||||
&["admin:video_tasks:write"]
|
||||
} else {
|
||||
&["admin:video_tasks:read"]
|
||||
},
|
||||
write,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:api_keys:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &[
|
||||
"admin:monitoring:admin",
|
||||
"admin:usage:read",
|
||||
"admin:api_keys:read",
|
||||
],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||
{
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:admin"],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:usage:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn management_token_has_operational_permission(
|
||||
permissions: &[String],
|
||||
required_permission: &str,
|
||||
) -> bool {
|
||||
let scope = required_permission
|
||||
.rsplit_once(':')
|
||||
.map(|(scope, _)| scope)
|
||||
.unwrap_or(required_permission);
|
||||
let admin_permission = format!("{scope}:admin");
|
||||
permissions
|
||||
.iter()
|
||||
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||
}
|
||||
|
||||
fn operational_auth_required_response() -> Response<Body> {
|
||||
let mut response = operational_error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"admin authentication required",
|
||||
None,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
http::header::WWW_AUTHENTICATE,
|
||||
HeaderValue::from_static("Bearer"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||
operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational permission denied",
|
||||
Some(required_permission),
|
||||
)
|
||||
}
|
||||
|
||||
fn operational_error_response(
|
||||
status: StatusCode,
|
||||
detail: &'static str,
|
||||
required_permission: Option<&'static str>,
|
||||
) -> Response<Body> {
|
||||
let mut response = (
|
||||
status,
|
||||
Json(json!({
|
||||
"detail": detail,
|
||||
"required_permission": required_permission,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayLocalAuthRejection;
|
||||
use crate::headers::should_skip_response_header;
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use crate::{insert_header_if_missing, GatewayError};
|
||||
|
||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
||||
);
|
||||
}
|
||||
|
||||
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||
// Provider responses are API data, even when an untrusted provider labels
|
||||
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||
// from becoming same-origin active content, and prevent referrer leakage
|
||||
// if a user follows a link rendered from such a response.
|
||||
headers.insert(
|
||||
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("content-security-policy"),
|
||||
HeaderValue::from_static(
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||
),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("referrer-policy"),
|
||||
HeaderValue::from_static("no-referrer"),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response(
|
||||
upstream_response: reqwest::Response,
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let status = upstream_response.status();
|
||||
let upstream_headers = upstream_response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
(
|
||||
name.as_str().to_string(),
|
||||
value.to_str().unwrap_or_default().to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||
let upstream_stream = upstream_response.bytes_stream();
|
||||
build_client_response_from_parts(
|
||||
status.as_u16(),
|
||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
headers
|
||||
.get_all(http::header::CONNECTION)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok()),
|
||||
);
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
let normalized = name.as_str().to_ascii_lowercase();
|
||||
if should_skip_client_response_header(&normalized)
|
||||
|| connection_declared.contains(&normalized)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|value| (normalized, value.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn should_skip_client_response_header(name: &str) -> bool {
|
||||
should_skip_response_header(name)
|
||||
// A provider Location is relative to the provider, not to the gateway.
|
||||
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||
// can disclose their gateway Authorization header to another origin.
|
||||
// Keep Location available inside execution reports, but never expose
|
||||
// it on the client-facing response boundary.
|
||||
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response_from_parts(
|
||||
status_code: u16,
|
||||
upstream_headers: &BTreeMap<String, String>,
|
||||
@@ -111,8 +158,17 @@ where
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
upstream_headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||
.map(|(_, value)| value.as_str()),
|
||||
);
|
||||
|
||||
for (name, value) in upstream_headers {
|
||||
if should_skip_response_header(name.as_str()) {
|
||||
if should_skip_client_response_header(name.as_str())
|
||||
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||
@@ -123,6 +179,7 @@ where
|
||||
}
|
||||
mutate_headers(response.headers_mut())?;
|
||||
apply_streaming_response_headers(response.headers_mut());
|
||||
apply_gateway_browser_security_headers(response.headers_mut());
|
||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||
if let Some(decision) = control_decision {
|
||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_plan_usage_limited_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
rejection: &PlanUsagePolicyRejection,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||
let fallback_payload = json!({
|
||||
"error": {
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": message,
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
});
|
||||
let payload = build_local_error_payload(
|
||||
control_decision,
|
||||
None,
|
||||
message,
|
||||
LocalCoreSyncErrorKind::RateLimit,
|
||||
fallback_payload,
|
||||
);
|
||||
let body =
|
||||
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let headers = BTreeMap::from([
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||
(
|
||||
"X-RateLimit-Metric".to_string(),
|
||||
rejection.metric.to_string(),
|
||||
),
|
||||
(
|
||||
"X-RateLimit-Window".to_string(),
|
||||
rejection.window.to_string(),
|
||||
),
|
||||
]);
|
||||
build_client_response_from_parts(
|
||||
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||
&headers,
|
||||
Body::from(body),
|
||||
trace_id,
|
||||
control_decision,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_http_error_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
||||
build_client_response, build_client_response_from_parts,
|
||||
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||
build_local_user_rpm_limited_response,
|
||||
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||
};
|
||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use axum::body::{to_bytes, Body};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -490,6 +600,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||
let response = build_client_response_from_parts_with_mutator(
|
||||
200,
|
||||
&BTreeMap::from([
|
||||
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||
(
|
||||
"x-aether-gateway".to_string(),
|
||||
"attacker-gateway".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-control-action".to_string(),
|
||||
"attacker-action".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-future-control".to_string(),
|
||||
"attacker-future".to_string(),
|
||||
),
|
||||
(
|
||||
"x-accel-redirect".to_string(),
|
||||
"/internal/private-file".to_string(),
|
||||
),
|
||||
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||
(
|
||||
"x-reproxy-url".to_string(),
|
||||
"http://127.0.0.1:9000/private".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-origin".to_string(),
|
||||
"https://attacker.example".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-credentials".to_string(),
|
||||
"true".to_string(),
|
||||
),
|
||||
("content-length".to_string(), "999999".to_string()),
|
||||
(
|
||||
"content-security-policy".to_string(),
|
||||
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||
),
|
||||
(
|
||||
"content-security-policy-report-only".to_string(),
|
||||
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||
),
|
||||
(
|
||||
"reporting-endpoints".to_string(),
|
||||
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||
),
|
||||
("report-to".to_string(), "attacker".to_string()),
|
||||
(
|
||||
"nel".to_string(),
|
||||
"{\"report_to\":\"attacker\"}".to_string(),
|
||||
),
|
||||
(
|
||||
"refresh".to_string(),
|
||||
"0; url=https://attacker.example".to_string(),
|
||||
),
|
||||
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||
(
|
||||
"location".to_string(),
|
||||
"https://provider.example/direct".to_string(),
|
||||
),
|
||||
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||
]),
|
||||
Body::empty(),
|
||||
"trace-upstream-header-filter",
|
||||
None,
|
||||
|headers| {
|
||||
headers.insert(
|
||||
http::HeaderName::from_static("x-aether-control-action"),
|
||||
http::HeaderValue::from_static("gateway-action"),
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("response should build");
|
||||
|
||||
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||
assert!(response.headers().get("x-sendfile").is_none());
|
||||
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("content-security-policy-report-only")
|
||||
.is_none());
|
||||
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||
assert!(response.headers().get("report-to").is_none());
|
||||
assert!(response.headers().get("nel").is_none());
|
||||
assert!(response.headers().get("refresh").is_none());
|
||||
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||
assert_eq!(
|
||||
response.headers()["content-security-policy"],
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||
);
|
||||
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||
assert_eq!(
|
||||
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||
"nosniff"
|
||||
);
|
||||
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||
assert_eq!(
|
||||
response.headers()["x-aether-control-action"],
|
||||
"gateway-action"
|
||||
);
|
||||
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener");
|
||||
let addr = listener.local_addr().expect("address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).await.expect("request read");
|
||||
stream
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||
)
|
||||
.await
|
||||
.expect("response write");
|
||||
});
|
||||
let upstream = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("client")
|
||||
.get(format!("http://{addr}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upstream response");
|
||||
|
||||
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||
.expect("client response");
|
||||
server.await.expect("server");
|
||||
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-first-hop").is_none());
|
||||
assert!(response.headers().get("x-second-hop").is_none());
|
||||
}
|
||||
|
||||
fn claude_decision() -> GatewayControlDecision {
|
||||
GatewayControlDecision::synthetic(
|
||||
"/v1/messages",
|
||||
@@ -581,4 +848,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||
let response = build_local_plan_usage_limited_response(
|
||||
"trace-plan-limit",
|
||||
None,
|
||||
&PlanUsagePolicyRejection {
|
||||
metric: "request_count",
|
||||
limit: 100.0,
|
||||
retry_after: 42,
|
||||
window: "calendar_week",
|
||||
},
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(response.headers()["retry-after"], "42");
|
||||
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||
let payload = response_json(response).await;
|
||||
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_contracts::ExecutionResult;
|
||||
@@ -21,7 +22,9 @@ use super::{
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
||||
pub(crate) use self::cancel::{
|
||||
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ListVideoTasksQuery {
|
||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
||||
}
|
||||
|
||||
pub(crate) async fn build_video_task_video_response(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
task_id: &str,
|
||||
source: VideoTaskVideoSource,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
match source {
|
||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
||||
VideoTaskVideoSource::Redirect { url } => {
|
||||
resolve_public_video_target(&url).await?;
|
||||
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||
}
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
||||
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
||||
}
|
||||
|
||||
async fn proxy_video_stream(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
url: &str,
|
||||
url: &url::Url,
|
||||
header_name: &str,
|
||||
header_value: &str,
|
||||
filename: &str,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
let response = state
|
||||
.client
|
||||
.get(url)
|
||||
let target = resolve_public_video_target(url).await?;
|
||||
let client = build_pinned_video_client(&target)?;
|
||||
let response = client
|
||||
.get(url.clone())
|
||||
.header(header_name, header_value)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: err.to_string(),
|
||||
message: video_request_failure_message(&err).to_string(),
|
||||
})?;
|
||||
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
if response.status().is_redirection() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!(
|
||||
"video upstream redirect was rejected with HTTP {}",
|
||||
response.status()
|
||||
),
|
||||
});
|
||||
}
|
||||
if !response.status().is_success() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!("video upstream returned HTTP {}", response.status()),
|
||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
||||
let content_length = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_LENGTH)
|
||||
.cloned();
|
||||
let cache_control = response
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.cloned();
|
||||
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||
// Reqwest may decode transfer/content encodings and the provider controls
|
||||
// the declaration; forwarding a stale value would make the client-facing
|
||||
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||
// will select safe framing for the actual stream.
|
||||
let upstream_headers = response.headers().clone();
|
||||
let body = Body::from_stream(response.bytes_stream());
|
||||
|
||||
let mut outbound = axum::http::Response::builder()
|
||||
.status(status)
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
if let Some(content_length) = content_length {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
||||
}
|
||||
if let Some(cache_control) = cache_control {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
||||
} else {
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
||||
);
|
||||
}
|
||||
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||
Ok(outbound)
|
||||
}
|
||||
|
||||
fn apply_safe_video_response_metadata(
|
||||
outbound: &mut axum::http::HeaderMap,
|
||||
upstream: &axum::http::HeaderMap,
|
||||
filename: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
let content_type = upstream
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(safe_video_content_type)
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!(
|
||||
"inline; filename=\"{}\"",
|
||||
safe_video_filename(filename)
|
||||
))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||
outbound.insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
outbound.insert(
|
||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
axum::http::HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||
let subtype = media_type.strip_prefix("video/")?;
|
||||
if subtype.is_empty()
|
||||
|| !subtype.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||
)
|
||||
})
|
||||
{
|
||||
return None;
|
||||
}
|
||||
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||
}
|
||||
|
||||
fn safe_video_filename(filename: &str) -> String {
|
||||
let filename = filename
|
||||
.chars()
|
||||
.take(255)
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||
character
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
if filename.is_empty() {
|
||||
"video.mp4".to_string()
|
||||
} else {
|
||||
filename
|
||||
}
|
||||
}
|
||||
|
||||
struct ResolvedVideoTarget {
|
||||
host: String,
|
||||
addrs: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||
));
|
||||
}
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||
let (host, addrs) = match url.host() {
|
||||
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||
let addrs = aether_http::lookup_host_with_limits(
|
||||
host,
|
||||
port,
|
||||
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||
(host.to_string(), addrs)
|
||||
}
|
||||
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||
};
|
||||
if addrs.is_empty()
|
||||
|| addrs
|
||||
.iter()
|
||||
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL resolves to a private or reserved address",
|
||||
));
|
||||
}
|
||||
Ok(ResolvedVideoTarget { host, addrs })
|
||||
}
|
||||
|
||||
fn build_pinned_video_client(
|
||||
target: &ResolvedVideoTarget,
|
||||
) -> Result<reqwest::Client, GatewayError> {
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(10_000),
|
||||
request_timeout_ms: Some(300_000),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if target.host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||
}
|
||||
builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||
}
|
||||
|
||||
fn video_target_rejected(message: &str) -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"video upstream request timed out"
|
||||
} else if error.is_connect() {
|
||||
"video upstream connection failed"
|
||||
} else if error.is_body() || error.is_decode() {
|
||||
"video upstream response failed"
|
||||
} else {
|
||||
"video upstream request failed"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use super::{
|
||||
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_accepts_public_target() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let target = "https://8.8.8.8/video.mp4";
|
||||
|
||||
let response = build_video_task_video_response(
|
||||
&state,
|
||||
"task-public-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(target).expect("public target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("public redirect should build");
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(axum::http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(target)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
] {
|
||||
let error = build_video_task_video_response(
|
||||
&state,
|
||||
"task-rejected-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("private or reserved redirect target should be rejected");
|
||||
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
axum::http::StatusCode::BAD_GATEWAY,
|
||||
"unexpected status for {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_err(),
|
||||
"target should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||
for raw_url in [
|
||||
"https://8.8.8.8/video.mp4",
|
||||
"https://[2606:4700:4700::1111]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_ok(),
|
||||
"target should be accepted: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||
assert!(safe_video_content_type("video/mp4").is_some());
|
||||
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||
assert!(safe_video_content_type("video/").is_none());
|
||||
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||
assert!(safe_video_content_type("text/html").is_none());
|
||||
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||
assert_eq!(
|
||||
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||
"video_123.mp4___filename__attack.html"
|
||||
);
|
||||
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||
|
||||
let mut upstream = axum::http::HeaderMap::new();
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
axum::http::HeaderValue::from_static("text/html"),
|
||||
);
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_LENGTH,
|
||||
axum::http::HeaderValue::from_static("999999"),
|
||||
);
|
||||
let mut outbound = upstream.clone();
|
||||
apply_safe_video_response_metadata(
|
||||
&mut outbound,
|
||||
&upstream,
|
||||
"video.mp4\"; filename=\"attack.html",
|
||||
)
|
||||
.expect("video metadata should build");
|
||||
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("application/octet-stream")
|
||||
);
|
||||
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("nosniff")
|
||||
);
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::finalize_video_task_if_terminal;
|
||||
use super::super::read_video_task_detail;
|
||||
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||
use super::current_unix_secs;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
cancel_video_task_record_inner(state, task_id, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn cancel_video_task_record_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let user_id = user_id.trim();
|
||||
if user_id.is_empty() {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||
}
|
||||
|
||||
async fn cancel_video_task_record_inner(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
expected_user_id: Option<&str>,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let task = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(task) = task else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
|
||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
||||
}
|
||||
|
||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||
let mut finalize_mutation = None;
|
||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||
state
|
||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
||||
.await?;
|
||||
|
||||
let body_json = json!({});
|
||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
);
|
||||
let follow_up = if let Some(user_id) = expected_user_id {
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
user_id,
|
||||
task.api_key_id.as_deref(),
|
||||
&trace_id,
|
||||
)
|
||||
} else {
|
||||
state
|
||||
.hydrate_video_task_for_route(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
)
|
||||
.await?;
|
||||
state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(follow_up) = follow_up {
|
||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||
.await
|
||||
.map_err(CancelVideoTaskError::Response)?;
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
} else if expected_user_id.is_none() {
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
||||
}
|
||||
|
||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||
Some(stored) => stored,
|
||||
None => {
|
||||
let current = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(current) = current else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
if !current.status.is_active() {
|
||||
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||
}
|
||||
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
"video task repository is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
)));
|
||||
}
|
||||
};
|
||||
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&request_path, &report_kind);
|
||||
}
|
||||
finalize_video_task_if_terminal(state, &stored).await;
|
||||
Ok(stored)
|
||||
}
|
||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
||||
}
|
||||
|
||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||
let provider_api_format = task
|
||||
.provider_api_format
|
||||
.as_deref()
|
||||
.or(task.client_api_format.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let provider_api_format = task.effective_api_format()?;
|
||||
|
||||
match provider_api_format {
|
||||
"openai:video" => Some(VideoTaskCancelPlan {
|
||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
||||
let result =
|
||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
.map_err(|_| {
|
||||
GatewayError::UpstreamUnavailable {
|
||||
trace_id: trace_id.to_string(),
|
||||
message: format!("{err:?}"),
|
||||
message: "video cancellation request failed".to_string(),
|
||||
}
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
if result.status_code >= 400 {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
let body_json = result
|
||||
.body
|
||||
.and_then(|body| body.json_body)
|
||||
.unwrap_or_else(|| {
|
||||
json!({
|
||||
"error": {
|
||||
"message": result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.unwrap_or_else(|| {
|
||||
format!("execution runtime returned {}", result.status_code)
|
||||
}),
|
||||
}
|
||||
})
|
||||
});
|
||||
return Err((status, Json(body_json)).into_response());
|
||||
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_cancelled_request_metadata(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<Value>, GatewayError> {
|
||||
let mut metadata = match task.request_metadata.clone() {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
||||
if snapshot_value.is_none() {
|
||||
snapshot_value = state
|
||||
.reconstruct_video_task_snapshot(task)
|
||||
.await?
|
||||
.map(|snapshot| {
|
||||
serde_json::to_value(snapshot)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
}
|
||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
snapshot_value_ref.clone(),
|
||||
);
|
||||
return Ok(Some(Value::Object(metadata)));
|
||||
}
|
||||
|
||||
Ok(task.request_metadata.clone())
|
||||
}
|
||||
|
||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("OpenAi")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
return;
|
||||
}
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("Gemini")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
}
|
||||
fn build_video_task_cancel_upstream_error_response(
|
||||
result: &aether_contracts::ExecutionResult,
|
||||
) -> axum::response::Response {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
tracing::warn!(
|
||||
event_name = "video_task_cancel_upstream_error",
|
||||
upstream_status = result.status_code,
|
||||
"video cancellation upstream response body discarded"
|
||||
);
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": format!(
|
||||
"video cancellation upstream returned HTTP {}",
|
||||
result.status_code
|
||||
),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn persist_cancelled_video_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
request_metadata: Option<Value>,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
let now_unix_secs = current_unix_secs();
|
||||
state
|
||||
.data
|
||||
.upsert_video_task(UpsertVideoTask {
|
||||
.update_active_video_task(UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
request_id: task.request_id.clone(),
|
||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: task.original_request_body.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: VideoTaskStatus::Cancelled,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds,
|
||||
next_poll_at_unix_secs: None,
|
||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
||||
completed_at_unix_secs: Some(now_unix_secs),
|
||||
updated_at_unix_secs: now_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata,
|
||||
request_metadata: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||
};
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
|
||||
use super::build_video_task_cancel_upstream_error_response;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "cancel-secret-request-id".to_string(),
|
||||
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||
status_code: 502,
|
||||
headers: BTreeMap::from([(
|
||||
"x-internal-secret".to_string(),
|
||||
"cancel-secret-header".to_string(),
|
||||
)]),
|
||||
response_observation: None,
|
||||
body: Some(ResponseBody {
|
||||
json_body: Some(json!({
|
||||
"error": {
|
||||
"message": "cancel-secret-upstream-body",
|
||||
}
|
||||
})),
|
||||
body_bytes_b64: None,
|
||||
}),
|
||||
telemetry: None,
|
||||
error: Some(ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::FirstByte,
|
||||
message: "cancel-secret-runtime-error".to_string(),
|
||||
upstream_status: Some(502),
|
||||
retryable: true,
|
||||
failover_recommended: false,
|
||||
}),
|
||||
};
|
||||
|
||||
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||
assert!(response.headers().get("x-internal-secret").is_none());
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("response body should parse");
|
||||
|
||||
assert_eq!(
|
||||
payload,
|
||||
json!({
|
||||
"error": {
|
||||
"message": "video cancellation upstream returned HTTP 502",
|
||||
}
|
||||
})
|
||||
);
|
||||
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||
assert!(!body.contains("cancel-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||
pub(crate) use http::{
|
||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
||||
CancelVideoTaskError,
|
||||
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||
};
|
||||
pub(crate) use query::{
|
||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
||||
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
pub(crate) use runtime::{
|
||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||
|
||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
||||
pub(crate) processing_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum VideoTaskVideoSource {
|
||||
Redirect {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
},
|
||||
Proxy {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
header_name: String,
|
||||
header_value: String,
|
||||
filename: String,
|
||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
||||
state.find_video_task_by_id(task_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_detail_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_video_source(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
video_task_video_source_from_task(state, &task).await
|
||||
}
|
||||
|
||||
pub(crate) async fn video_task_video_source_from_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||
let Some(video_url) = task
|
||||
.video_url
|
||||
.as_deref()
|
||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
||||
let video_url = parse_video_url(&video_url)?;
|
||||
|
||||
if task.effective_api_format() != Some("gemini:video") {
|
||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||
}
|
||||
|
||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
||||
));
|
||||
};
|
||||
|
||||
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||
})?;
|
||||
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||
});
|
||||
}
|
||||
let api_key = transport.key.decrypted_api_key.trim();
|
||||
if api_key.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
||||
url: video_url,
|
||||
header_name: "x-goog-api-key".to_string(),
|
||||
header_value: api_key.to_string(),
|
||||
filename: format!("video_{task_id}.mp4"),
|
||||
filename: format!("video_{}.mp4", task.id),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL is invalid".to_string(),
|
||||
})?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||
left.scheme() == right.scheme()
|
||||
&& left.host() == right.host()
|
||||
&& left.port_or_known_default() == right.port_or_known_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_stats(
|
||||
state: &AppState,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||
now_unix_secs - (now_unix_secs % 86_400)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::{data::GatewayDataState, AppState};
|
||||
|
||||
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||
{
|
||||
UpsertVideoTask {
|
||||
id: "legacy-gemini-task".to_string(),
|
||||
short_id: Some("legacy-short".to_string()),
|
||||
request_id: "legacy-request".to_string(),
|
||||
user_id: Some("user-1".to_string()),
|
||||
api_key_id: Some("client-key-1".to_string()),
|
||||
username: None,
|
||||
api_key_name: None,
|
||||
external_task_id: Some("operations/upstream-1".to_string()),
|
||||
provider_id: Some("provider-1".to_string()),
|
||||
endpoint_id: Some("endpoint-1".to_string()),
|
||||
key_id: Some("provider-key-1".to_string()),
|
||||
client_api_format: Some("gemini:video".to_string()),
|
||||
provider_api_format: None,
|
||||
format_converted: false,
|
||||
model: Some("veo-3".to_string()),
|
||||
prompt: None,
|
||||
original_request_body: None,
|
||||
duration_seconds: Some(8),
|
||||
resolution: Some("720p".to_string()),
|
||||
aspect_ratio: Some("16:9".to_string()),
|
||||
size: Some("1280x720".to_string()),
|
||||
status: VideoTaskStatus::Completed,
|
||||
progress_percent: 100,
|
||||
progress_message: None,
|
||||
retry_count: 0,
|
||||
poll_interval_seconds: 10,
|
||||
next_poll_at_unix_secs: None,
|
||||
poll_count: 1,
|
||||
max_poll_count: 360,
|
||||
created_at_unix_ms: 1,
|
||||
submitted_at_unix_secs: Some(1),
|
||||
completed_at_unix_secs: Some(2),
|
||||
updated_at_unix_secs: 2,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
video_url: Some(
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
.to_string(),
|
||||
),
|
||||
request_metadata: None,
|
||||
}
|
||||
.into_stored()
|
||||
}
|
||||
|
||||
fn state_with_gemini_transport() -> AppState {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let provider = StoredProviderCatalogProvider::new(
|
||||
"provider-1".to_string(),
|
||||
"Gemini".to_string(),
|
||||
Some("https://ai.google.dev".to_string()),
|
||||
"gemini".to_string(),
|
||||
)
|
||||
.expect("provider should build");
|
||||
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||
"endpoint-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"gemini:video".to_string(),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("endpoint should build")
|
||||
.with_transport_fields(
|
||||
"https://generativelanguage.googleapis.com".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let encrypted_api_key = state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-1",
|
||||
"provider-key-1",
|
||||
"gemini-provider-secret",
|
||||
)
|
||||
.expect("provider key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
"provider-key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"default".to_string(),
|
||||
"api_key".to_string(),
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("provider key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["gemini:video"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("provider key transport should build");
|
||||
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||
);
|
||||
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||
video_tasks,
|
||||
provider_catalog,
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
state.with_data_state_for_tests(data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||
for raw_url in [
|
||||
"file:///etc/passwd",
|
||||
"data:video/mp4;base64,AAAA",
|
||||
"https://[email protected]/video.mp4",
|
||||
"https://user:[email protected]/video.mp4",
|
||||
"/relative/video.mp4",
|
||||
] {
|
||||
assert!(
|
||||
parse_video_url(raw_url).is_err(),
|
||||
"URL should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||
for same_origin in [
|
||||
"https://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:443/file",
|
||||
] {
|
||||
assert!(video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(same_origin).unwrap()
|
||||
));
|
||||
}
|
||||
for different_origin in [
|
||||
"http://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:444/file",
|
||||
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||
] {
|
||||
assert!(!video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(different_origin).unwrap()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||
let source = video_task_video_source_from_task(
|
||||
&state_with_gemini_transport(),
|
||||
&legacy_gemini_video_task(),
|
||||
)
|
||||
.await
|
||||
.expect("video source should resolve")
|
||||
.expect("video source should exist");
|
||||
|
||||
match source {
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => {
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
);
|
||||
assert_eq!(header_name, "x-goog-api-key");
|
||||
assert_eq!(header_value, "gemini-provider-secret");
|
||||
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||
}
|
||||
VideoTaskVideoSource::Redirect { .. } => {
|
||||
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VideoTaskRefreshError {
|
||||
message: String,
|
||||
category: &'static str,
|
||||
permanent: bool,
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
||||
warn!(
|
||||
event_name = "video_task_refresh_failed",
|
||||
log_type = "event",
|
||||
error = %err.message,
|
||||
error_category = err.category,
|
||||
permanent = err.permanent,
|
||||
"gateway video task refresh failed"
|
||||
);
|
||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let mut refreshed = 0usize;
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
let trace_id = format!("video-task-poller-{index}");
|
||||
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||
continue;
|
||||
};
|
||||
let Some(refresh_plan) = state
|
||||
.video_tasks
|
||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
||||
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||
let Some(updated) =
|
||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
||||
let Some(updated) = build_successful_poll_update(
|
||||
&task,
|
||||
snapshot.clone(),
|
||||
&provider_body,
|
||||
now_unix_secs,
|
||||
)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: format!("{err:?}"),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
}));
|
||||
}
|
||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.and_then(|body| body.as_object().cloned())
|
||||
else {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: "video task refresh missing json provider body".to_string(),
|
||||
category: "invalid_provider_response",
|
||||
permanent: false,
|
||||
}));
|
||||
};
|
||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
.as_ref()
|
||||
.and_then(|error| error.upstream_status)
|
||||
.unwrap_or(result.status_code);
|
||||
let message = result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.or_else(|| {
|
||||
result
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|body| body.json_body.as_ref())
|
||||
.and_then(|value| value.get("error"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
||||
let category = if status_code == 401 {
|
||||
"authentication_error"
|
||||
} else if status_code == 403 {
|
||||
"permission_denied"
|
||||
} else if status_code == 404 {
|
||||
"not_found"
|
||||
} else if status_code == 429 {
|
||||
"rate_limit"
|
||||
} else if status_code >= 500 {
|
||||
"server_error"
|
||||
} else {
|
||||
"provider_error"
|
||||
};
|
||||
let permanent = result.error.as_ref().map_or(
|
||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||
|error| match error.kind {
|
||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
},
|
||||
);
|
||||
|
||||
VideoTaskRefreshError { message, permanent }
|
||||
VideoTaskRefreshError {
|
||||
category,
|
||||
permanent,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_successful_poll_update(
|
||||
task: &StoredVideoTask,
|
||||
mut snapshot: LocalVideoTaskSnapshot,
|
||||
provider_body: &Map<String, Value>,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
||||
return Ok(None);
|
||||
};
|
||||
snapshot.apply_provider_body(provider_body);
|
||||
|
||||
let mut record = snapshot.to_upsert_record();
|
||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
||||
record.format_converted = task.format_converted;
|
||||
record.model = task.model.clone().or(record.model);
|
||||
record.prompt = task.prompt.clone().or(record.prompt);
|
||||
record.original_request_body = task
|
||||
.original_request_body
|
||||
.clone()
|
||||
.or(record.original_request_body);
|
||||
record.original_request_body = None;
|
||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||
record.resolution = task.resolution.clone().or(record.resolution);
|
||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
Some(provider_body),
|
||||
None,
|
||||
)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
record.request_metadata = None;
|
||||
|
||||
Ok(Some(record))
|
||||
}
|
||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
||||
let mut record = stored_task_to_upsert(task);
|
||||
record.updated_at_unix_secs = now_unix_secs;
|
||||
record.poll_count = task.poll_count.saturating_add(1);
|
||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
||||
record.progress_message = None;
|
||||
if err.permanent {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_permanent_error".to_string());
|
||||
record.error_message = Some(err.message.clone());
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
} else {
|
||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| {
|
||||
merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
None,
|
||||
Some(err),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or(task.request_metadata.clone());
|
||||
record.request_metadata = None;
|
||||
record
|
||||
}
|
||||
|
||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
let snapshot_record =
|
||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
||||
UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
provider_api_format: task.provider_api_format.clone(),
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.prompt.clone())
|
||||
}),
|
||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.original_request_body.clone())
|
||||
}),
|
||||
duration_seconds: task.duration_seconds.or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.duration_seconds)
|
||||
}),
|
||||
resolution: task.resolution.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.resolution.clone())
|
||||
}),
|
||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.aspect_ratio.clone())
|
||||
}),
|
||||
size: task.size.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.size.clone())
|
||||
}),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: task.status,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||
}
|
||||
|
||||
fn merge_video_task_request_metadata(
|
||||
existing: Option<Value>,
|
||||
snapshot: &LocalVideoTaskSnapshot,
|
||||
provider_body: Option<&Map<String, Value>>,
|
||||
poll_error: Option<&VideoTaskRefreshError>,
|
||||
) -> Result<Option<Value>, serde_json::Error> {
|
||||
let mut metadata = match existing {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
serde_json::to_value(snapshot)?,
|
||||
);
|
||||
if let Some(provider_body) = provider_body {
|
||||
metadata.insert(
|
||||
"poll_raw_response".to_string(),
|
||||
Value::Object(provider_body.clone()),
|
||||
);
|
||||
metadata.remove("poll_error");
|
||||
}
|
||||
if let Some(poll_error) = poll_error {
|
||||
metadata.insert(
|
||||
"poll_error".to_string(),
|
||||
serde_json::json!({
|
||||
"message": poll_error.message,
|
||||
"permanent": poll_error.permanent,
|
||||
"observed_at_unix_secs": now_unix_secs(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(Some(Value::Object(metadata)))
|
||||
}
|
||||
|
||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||
return;
|
||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
||||
.or_else(|| task.provider_id.clone())
|
||||
let provider_name = task
|
||||
.provider_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let response_time_ms = task
|
||||
.submitted_at_unix_secs
|
||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
has_format_conversion: Some(task.format_converted),
|
||||
is_stream: Some(false),
|
||||
status_code,
|
||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
||||
error_message: task.error_code.clone(),
|
||||
response_time_ms,
|
||||
request_body: task.original_request_body.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_body: None,
|
||||
request_metadata: None,
|
||||
..UsageEventData::default()
|
||||
},
|
||||
))
|
||||
@@ -609,6 +536,9 @@ mod tests {
|
||||
|
||||
fn sample_sparse_stored_task() -> StoredVideoTask {
|
||||
let snapshot = LocalVideoTaskSnapshot::OpenAi(OpenAiVideoTaskSeed {
|
||||
local_short_id: None,
|
||||
native_response: None,
|
||||
xai_provider: false,
|
||||
local_task_id: "task-1".to_string(),
|
||||
upstream_task_id: "ext-1".to_string(),
|
||||
created_at_unix_ms: 1,
|
||||
@@ -701,48 +631,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
||||
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.duration_seconds, Some(4));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.duration_seconds.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.aspect_ratio.is_none());
|
||||
assert!(record.size.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
||||
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||
let record = build_failed_poll_update(
|
||||
&sample_sparse_stored_task(),
|
||||
&VideoTaskRefreshError {
|
||||
message: "temporary failure".to_string(),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
},
|
||||
100,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
||||
path_and_query: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) {
|
||||
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||
let Some(decision) = control_decision else {
|
||||
return;
|
||||
};
|
||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
||||
},
|
||||
route_kind,
|
||||
default_target_type(route_family),
|
||||
path_and_query.to_string(),
|
||||
sanitized_path_and_query.clone(),
|
||||
)
|
||||
};
|
||||
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||
|
||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||
if log_level == AdminAuditLogLevel::Info {
|
||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||
if target_id.trim_start().starts_with('/') {
|
||||
return sanitize_admin_audit_path(&target_id);
|
||||
}
|
||||
target_id
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AdminAuditLogLevel {
|
||||
Info,
|
||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
||||
use super::{
|
||||
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||
AdminAuditLogLevel,
|
||||
};
|
||||
use axum::http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
@@ -169,4 +185,32 @@ mod tests {
|
||||
("failed", AdminAuditLogLevel::Warn)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_paths_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path(
|
||||
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||
),
|
||||
"/api/admin/providers?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||
"/install/[redacted]?view=raw"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id(
|
||||
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||
),
|
||||
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||
"resource-id?literal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
|
||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||
use super::scopes::BackupScope;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct S3BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) scope: BackupScope,
|
||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
||||
pub(crate) retention_count: u32,
|
||||
}
|
||||
|
||||
impl fmt::Debug for S3BackupConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||
formatter
|
||||
.debug_struct("S3BackupConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field("scope", &self.scope)
|
||||
.field("endpoint_origin", &endpoint_origin)
|
||||
.field("region", &self.region)
|
||||
.field("user_agent", &self.user_agent)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||
.field("path_style", &self.path_style)
|
||||
.field("compression", &self.compression)
|
||||
.field("schedule", &self.schedule)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct BackupConfigError {
|
||||
message: String,
|
||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
||||
"Endpoint(S3 地址)",
|
||||
enabled,
|
||||
)?;
|
||||
if enabled {
|
||||
validate_s3_endpoint(&endpoint)?;
|
||||
}
|
||||
let bucket =
|
||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||
let access_key_id = required_or_disabled_string(
|
||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
||||
enabled,
|
||||
)?;
|
||||
|
||||
let prefix = normalize_s3_prefix(
|
||||
&optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
scope,
|
||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||
bucket,
|
||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
prefix,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||
let prefix = prefix.trim().trim_matches('/');
|
||||
if prefix.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
if prefix
|
||||
.split('/')
|
||||
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||
|| prefix.contains('\\')
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||
let parsed = url::Url::parse(endpoint)
|
||||
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||
if parsed.scheme() != "https"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||
url::Url::parse(endpoint)
|
||||
.ok()
|
||||
.map(|parsed| parsed.origin().ascii_serialization())
|
||||
.unwrap_or_else(|| "<invalid>".to_string())
|
||||
}
|
||||
|
||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||
if (min..=max).contains(&value) {
|
||||
Ok(())
|
||||
@@ -374,6 +445,69 @@ mod tests {
|
||||
assert!(err.to_string().contains("Endpoint"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||
for endpoint in [
|
||||
"http://s3.example.com",
|
||||
"https://user:[email protected]",
|
||||
"https://s3.example.com?token=secret",
|
||||
"https://s3.example.com/#fragment",
|
||||
] {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": endpoint,
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
|
||||
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect_err("unsafe endpoint should fail closed");
|
||||
assert!(error.to_string().contains("Endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_output_does_not_expose_s3_credentials() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access-key-value",
|
||||
"backup_s3_secret_access_key": "secret-key-value"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("config should parse");
|
||||
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains("https://s3.example.com"));
|
||||
assert!(!debug.contains("/path"));
|
||||
assert!(!debug.contains("access-key-value"));
|
||||
assert!(!debug.contains("secret-key-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_s3_backup_prefix_once() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_prefix": "/prod/backups//",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("prefix should be canonicalized");
|
||||
|
||||
assert_eq!(config.prefix, "prod/backups/");
|
||||
|
||||
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||
let mut entries = entries.clone();
|
||||
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_default_values_from_system_config_contract() {
|
||||
let entries = serde_json::json!({
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
||||
pub(crate) mod task;
|
||||
pub(crate) mod worker;
|
||||
|
||||
pub use executor::{
|
||||
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||
};
|
||||
|
||||
use axum::body::Bytes;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupRestoreScope {
|
||||
Config,
|
||||
Users,
|
||||
Data,
|
||||
}
|
||||
|
||||
impl BackupRestoreScope {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "config",
|
||||
Self::Users => "users",
|
||||
Self::Data => "data",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("backup database apply failed: {0}")]
|
||||
pub struct BackupApplyError(String);
|
||||
|
||||
pub async fn apply_restored_backup(
|
||||
app: &crate::AppState,
|
||||
restored: RestoredBackupJson,
|
||||
scope: BackupRestoreScope,
|
||||
operator_id: Option<&str>,
|
||||
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||
if authority.scope() != scope {
|
||||
return Err(BackupApplyError(format!(
|
||||
"authenticated {} backup cannot be applied to {} scope",
|
||||
authority.scope().as_str(),
|
||||
scope.as_str(),
|
||||
)));
|
||||
}
|
||||
let request_body = Bytes::from(json_bytes);
|
||||
let state = crate::admin_api::AdminAppState::new(app);
|
||||
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupRestoreScope::Config => {
|
||||
state
|
||||
.restore_admin_system_config_backup(&request_body, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Users => {
|
||||
state
|
||||
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Data => {
|
||||
state
|
||||
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error {
|
||||
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||
"another system import or restore is already running"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||
"system import coordination is unavailable"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||
}
|
||||
};
|
||||
BackupApplyError(message.to_string())
|
||||
})?;
|
||||
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||
}
|
||||
|
||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||
RestoredBackupJson,
|
||||
};
|
||||
use crate::backup::executor::encrypt_backup_bytes;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use serde_json::json;
|
||||
|
||||
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||
let compressed = zstd::stream::encode_all(
|
||||
serde_json::to_vec(&json!({
|
||||
"version": "1.5",
|
||||
"exported_at": "2026-08-30T12:00:00Z",
|
||||
"users": [],
|
||||
"standalone_keys": [],
|
||||
}))
|
||||
.expect("test backup should serialize")
|
||||
.as_slice(),
|
||||
0,
|
||||
)
|
||||
.expect("test backup should compress");
|
||||
let (envelope, _) =
|
||||
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||
.expect("test backup should encrypt");
|
||||
super::restore_backup_json(
|
||||
object_key,
|
||||
&envelope,
|
||||
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.expect("test restore key should build")],
|
||||
BackupRestoreLimits::default(),
|
||||
)
|
||||
.expect("test backup should authenticate")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||
let restored = authenticated_users_backup();
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&crate::AppState::new().expect("test state should build"),
|
||||
restored,
|
||||
BackupRestoreScope::Config,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("scope mismatch must fail before database access");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||
let app = crate::AppState::new().expect("test state should build");
|
||||
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the shared import lease");
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&app,
|
||||
authenticated_users_backup(),
|
||||
BackupRestoreScope::Users,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("restore must not interleave with another system import");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: another system import or restore is already running"
|
||||
);
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::fmt;
|
||||
|
||||
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupScope {
|
||||
Config,
|
||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||
if object_key.is_empty()
|
||||
|| object_key.starts_with('/')
|
||||
|| object_key.contains('\0')
|
||||
|| object_key.contains('\\')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut segments = object_key.split('/').peekable();
|
||||
let mut file_name = None;
|
||||
while let Some(segment) = segments.next() {
|
||||
if segment.is_empty()
|
||||
|| segment == "."
|
||||
|| segment == ".."
|
||||
|| segment.chars().any(char::is_control)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if segments.peek().is_none() {
|
||||
file_name = Some(segment);
|
||||
}
|
||||
}
|
||||
let file_name = file_name?;
|
||||
|
||||
[Self::Config, Self::Users, Self::Data]
|
||||
.into_iter()
|
||||
.find(|scope| {
|
||||
file_name
|
||||
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||
.is_some_and(is_aether_backup_object_id)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn matching_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[
|
||||
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn matching_encrypted_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||
}
|
||||
|
||||
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||
)
|
||||
}
|
||||
|
||||
fn matching_backup_keys_with_suffixes(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
file_suffixes: &[&str],
|
||||
) -> Vec<String> {
|
||||
let normalized_prefix = normalized_prefix(prefix);
|
||||
let expected_prefix = if normalized_prefix.is_empty() {
|
||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
||||
format!("{normalized_prefix}/")
|
||||
};
|
||||
let file_prefix = format!("{}-", self.file_stem());
|
||||
let file_suffix = ".json.zst";
|
||||
|
||||
keys.into_iter()
|
||||
.filter(|key| {
|
||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
||||
if file_name.contains('/') {
|
||||
return false;
|
||||
}
|
||||
let Some(timestamp) = file_name
|
||||
.strip_prefix(&file_prefix)
|
||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
||||
else {
|
||||
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||
file_suffixes
|
||||
.iter()
|
||||
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(timestamp)
|
||||
is_aether_backup_object_id(timestamp)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn file_name(self, timestamp: &str) -> String {
|
||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
||||
format!(
|
||||
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||
self.file_stem()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||
if is_aether_backup_timestamp(value) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||
let (time, digest) = rest.split_once('-')?;
|
||||
Some((format!("{date}-{time}"), digest))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(×tamp)
|
||||
&& collision_digest.len() == 64
|
||||
&& collision_digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::BackupScope;
|
||||
@@ -130,15 +226,15 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
||||
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
||||
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -146,7 +242,7 @@ mod tests {
|
||||
fn retention_filter_only_matches_same_scope() {
|
||||
let keys = vec![
|
||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/random.json.zst".to_string(),
|
||||
];
|
||||
@@ -155,14 +251,18 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_filter_requires_aether_timestamp_format() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
let keys = vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||
@@ -171,13 +271,19 @@ mod tests {
|
||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||
];
|
||||
|
||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,11 +291,11 @@ mod tests {
|
||||
fn backup_key_prefix_boundaries_are_exact() {
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("", "20260524-031500"),
|
||||
"aether-config-backup-20260524-031500.json.zst"
|
||||
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
|
||||
let keys = vec![
|
||||
@@ -208,4 +314,36 @@ mod tests {
|
||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(
|
||||
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||
),
|
||||
Some(BackupScope::Config)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(&format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
)),
|
||||
Some(BackupScope::Users)
|
||||
);
|
||||
for key in [
|
||||
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||
] {
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(key),
|
||||
None,
|
||||
"unsafe or unrelated key: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::TryStreamExt;
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use object_store::path::Path;
|
||||
use object_store::{ClientOptions, ObjectStore};
|
||||
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||
use reqwest::header::HeaderValue;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupObjectCreateResult {
|
||||
Created,
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError>;
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let mut objects = self.objects.write().await;
|
||||
if objects.contains_key(key) {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
} else {
|
||||
objects.insert(key.to_string(), bytes);
|
||||
Ok(BackupObjectCreateResult::Created)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let bytes = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.get(key)
|
||||
.cloned()
|
||||
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||
if bytes.len() > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix = directory_list_prefix(prefix);
|
||||
Ok(self
|
||||
let keys: Vec<_> = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.starts_with(&prefix))
|
||||
.cloned()
|
||||
.collect())
|
||||
.collect();
|
||||
if keys.len() > max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
#[cfg(test)]
|
||||
impl FakeBackupObjectStore {
|
||||
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||
self.objects.read().await.get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut keys = self
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let options = PutOptions {
|
||||
mode: PutMode::Create,
|
||||
..PutOptions::default()
|
||||
};
|
||||
match self
|
||||
.store
|
||||
.list(prefix_path.as_ref())
|
||||
.map_ok(|meta| meta.location.to_string())
|
||||
.try_collect::<Vec<_>>()
|
||||
.put_opts(&Path::from(key), bytes.into(), options)
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
{
|
||||
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
}
|
||||
Err(error) => Err(BackupStoreError::object_store(
|
||||
"conditional put",
|
||||
key,
|
||||
error,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let result = self
|
||||
.store
|
||||
.get(&Path::from(key))
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
let object_size = result.meta.size;
|
||||
let mut stream = result.into_stream();
|
||||
let mut bytes = BytesMut::with_capacity(
|
||||
usize::try_from(object_size)
|
||||
.unwrap_or(max_bytes)
|
||||
.min(max_bytes)
|
||||
.min(8 * 1024 * 1024),
|
||||
);
|
||||
while let Some(chunk) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes.freeze())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut objects = self.store.list(prefix_path.as_ref());
|
||||
let mut keys = Vec::new();
|
||||
while let Some(meta) = objects
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||
{
|
||||
if keys.len() >= max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
keys.push(meta.location.to_string());
|
||||
}
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_list_prefix(prefix: &str) -> String {
|
||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
||||
use super::{
|
||||
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
||||
async fn fake_backup_object_store_puts_and_lists() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object(
|
||||
@@ -186,17 +336,59 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
assert_eq!(keys.len(), 2);
|
||||
|
||||
store
|
||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||
assert_eq!(
|
||||
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||
bytes::Bytes::from_static(b"1234")
|
||||
);
|
||||
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::Created
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::AlreadyExists
|
||||
);
|
||||
assert_eq!(
|
||||
store.object_bytes(key).await.as_deref(),
|
||||
Some(b"first".as_slice())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -218,7 +410,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
keys,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_admin::system::admin_system_config_default_value;
|
||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
||||
use futures_util::FutureExt;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::warn;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||
use super::scopes::BackupScope;
|
||||
use super::store::ObjectStoreS3BackupStore;
|
||||
use crate::admin_api::AdminAppState;
|
||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
||||
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||
use crate::task_runtime::{
|
||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
||||
];
|
||||
|
||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
enum BackupLockRenewalFailure<E> {
|
||||
Lost,
|
||||
Backend(E),
|
||||
}
|
||||
|
||||
enum BackupLockRaceOutcome<T> {
|
||||
BackupCompleted(T),
|
||||
LeaseLost(Result<(), JoinError>),
|
||||
}
|
||||
|
||||
impl S3BackupTaskError {
|
||||
fn bad_request(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
||||
impl std::error::Error for S3BackupTaskError {}
|
||||
|
||||
impl From<GatewayError> for S3BackupTaskError {
|
||||
fn from(error: GatewayError) -> Self {
|
||||
Self::internal(format!("{error:?}"))
|
||||
fn from(_error: GatewayError) -> Self {
|
||||
warn!(
|
||||
error_category = "dependency_failed",
|
||||
"S3 backup dependency failed"
|
||||
);
|
||||
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
||||
) -> Value {
|
||||
let mut payload = json!({
|
||||
"scope": config.scope.as_config_value(),
|
||||
"bucket": config.bucket.clone(),
|
||||
"prefix": config.prefix.clone(),
|
||||
"compression": config.compression.clone(),
|
||||
"trigger": trigger,
|
||||
});
|
||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
||||
Some(100),
|
||||
Some("S3 备份任务异常退出".to_string()),
|
||||
None,
|
||||
Some("S3 backup task panicked".to_string()),
|
||||
Some("background_task_panicked".to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
||||
.await;
|
||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
||||
let result = run_s3_backup_once(&app, &config).await;
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||
.await
|
||||
{
|
||||
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => result,
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task lost its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||
match heartbeat_result {
|
||||
Ok(()) => warn!(
|
||||
run_id = %run_id,
|
||||
"S3 backup task stopped after losing its distributed lock"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
run_id = %run_id,
|
||||
error = %error,
|
||||
"S3 backup lock heartbeat task failed"
|
||||
),
|
||||
}
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务已停止",
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(result) => {
|
||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
||||
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||
warn!(
|
||||
error_category = "slot_record_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup slot record failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||
None,
|
||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
||||
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup slot record failed",
|
||||
Some(json!({ "error": format!("{error:?}") })),
|
||||
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup task failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务失败".to_string()),
|
||||
None,
|
||||
Some(error.to_string()),
|
||||
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup task failed",
|
||||
Some(json!({ "error": error.to_string() })),
|
||||
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||
backup: F,
|
||||
mut heartbeat: JoinHandle<()>,
|
||||
) -> BackupLockRaceOutcome<T>
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
tokio::pin!(backup);
|
||||
tokio::select! {
|
||||
biased;
|
||||
heartbeat_result = &mut heartbeat => {
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||
}
|
||||
result = &mut backup => {
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
BackupLockRaceOutcome::BackupCompleted(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn require_successful_backup_lock_renewal<E>(
|
||||
result: Result<bool, E>,
|
||||
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||
match result {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_s3_backup_task_heartbeat(
|
||||
app: AppState,
|
||||
run_id: String,
|
||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let _ = app
|
||||
.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await;
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task distributed lock is no longer owned"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task distributed lock renewal failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||
)),
|
||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
||||
"无法获取 S3 备份任务锁:{error}"
|
||||
))),
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "lock_acquisition_failed",
|
||||
"S3 backup task lock acquisition failed"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法获取 S3 备份任务锁,请稍后重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
||||
app: &AppState,
|
||||
config: &S3BackupConfig,
|
||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
let payload = match config.scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
||||
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||
return Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||
));
|
||||
};
|
||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now())
|
||||
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||
warn!(
|
||||
error_category = "object_store_initialization_failed",
|
||||
"S3 backup object store initialization failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||
.await
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
||||
.map_err(|_| {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
"S3 backup execution failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})
|
||||
}
|
||||
|
||||
async fn build_s3_backup_payload_exclusively(
|
||||
app: &AppState,
|
||||
scope: BackupScope,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Data => {
|
||||
admin_state
|
||||
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
warn!(
|
||||
error_category = "system_import_coordination_failed",
|
||||
lock_error = ?error,
|
||||
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||
);
|
||||
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?
|
||||
.map_err(S3BackupTaskError::from)
|
||||
}
|
||||
|
||||
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| {
|
||||
app.encryption_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_s3_backup_config_for_run(
|
||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
.or_else(|| admin_system_config_default_value(key));
|
||||
if let Some(value) = value {
|
||||
let value = if *key == "backup_s3_secret_access_key" {
|
||||
decrypt_s3_secret_access_key(app, value)?
|
||||
decrypt_s3_secret_access_key(app, value).await?
|
||||
} else {
|
||||
value
|
||||
};
|
||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(ciphertext) = value
|
||||
async fn decrypt_s3_secret_access_key(
|
||||
app: &AppState,
|
||||
value: Value,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(stored_value) = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Ok(value);
|
||||
};
|
||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
||||
else {
|
||||
return Err(S3BackupTaskError::bad_request(
|
||||
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||
app,
|
||||
"backup_s3_secret_access_key",
|
||||
stored_value.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
S3BackupTaskError::bad_request(
|
||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||
));
|
||||
};
|
||||
)
|
||||
})?;
|
||||
Ok(Value::String(plaintext))
|
||||
}
|
||||
|
||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||
json!({
|
||||
"scope": result.scope.as_config_value(),
|
||||
"bucket": result.bucket,
|
||||
"object_key": result.object_key,
|
||||
"bytes": result.bytes,
|
||||
"sha256": result.sha256,
|
||||
"export_version": result.export_version,
|
||||
"exported_at": result.exported_at,
|
||||
"compression": result.compression,
|
||||
"deleted_old_objects": result.deleted_old_objects,
|
||||
"encryption": result.encryption,
|
||||
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
@@ -603,9 +801,78 @@ mod tests {
|
||||
};
|
||||
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::handlers::shared::decrypt_system_config_secret;
|
||||
use crate::state::AppState;
|
||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||
|
||||
#[test]
|
||||
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||
Ok(())
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||
Err(super::BackupLockRenewalFailure::Lost)
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||
Err(super::BackupLockRenewalFailure::Backend(
|
||||
"redis unavailable"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||
let backup = async move {
|
||||
std::future::pending::<()>().await;
|
||||
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||
Ok::<(), super::S3BackupTaskError>(())
|
||||
};
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||
let app = AppState::new().expect("app state should build");
|
||||
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the system import lease");
|
||||
|
||||
let error = super::build_s3_backup_payload_exclusively(
|
||||
&app,
|
||||
crate::backup::scopes::BackupScope::Config,
|
||||
)
|
||||
.await
|
||||
.expect_err("backup snapshot must not overlap a system import");
|
||||
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
}
|
||||
|
||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||
vec![
|
||||
(
|
||||
@@ -632,6 +899,92 @@ mod tests {
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||
let plaintext = "legacy-s3-secret-access-key";
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(plaintext);
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let values = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect("legacy S3 config should load");
|
||||
assert_eq!(
|
||||
values.get("backup_s3_secret_access_key"),
|
||||
Some(&serde_json::json!(plaintext))
|
||||
);
|
||||
|
||||
let stored = app
|
||||
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read")
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.expect("stored S3 secret should remain a string");
|
||||
assert_ne!(stored, plaintext);
|
||||
assert_eq!(
|
||||
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||
.expect("migrated S3 secret should decrypt"),
|
||||
plaintext
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||
let plaintext = "s3-secret-from-unavailable-key";
|
||||
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||
.expect("unknown-key fixture should encrypt");
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(ciphertext.clone());
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let error = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||
let error_text = error.to_string();
|
||||
assert!(!error_text.contains(plaintext));
|
||||
assert!(!error_text.contains(&ciphertext));
|
||||
assert_eq!(
|
||||
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read"),
|
||||
Some(serde_json::json!(ciphertext))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||
"postgresql://admin:[email protected]/aether".to_string(),
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
error.status(),
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
assert!(!error.detail().contains("database-secret"));
|
||||
}
|
||||
|
||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||
let now = now_unix_secs();
|
||||
StoredBackgroundTaskRun {
|
||||
@@ -774,7 +1127,9 @@ mod tests {
|
||||
|
||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||
|
||||
assert!(payload["bucket"].is_string());
|
||||
assert!(payload.get("bucket").is_none());
|
||||
assert!(payload.get("prefix").is_none());
|
||||
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||
assert!(!payload.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
||||
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||
warn!(
|
||||
error_category = "schedule_tick_failed",
|
||||
"S3 backup schedule tick failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
||||
) -> Result<(), GatewayError> {
|
||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||
Ok(values) => values,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config load failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_load_failed",
|
||||
"S3 backup schedule config load failed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let config = match S3BackupConfig::from_json_map(&values) {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_invalid",
|
||||
"S3 backup schedule config is invalid"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
||||
|
||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "task_submission_failed",
|
||||
"S3 backup scheduled task submission failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||
system_config_bool, system_config_string,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use serde_json::{json, Value};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||
@@ -10,8 +12,19 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||
|
||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BarkPushConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) device_key: Option<String>,
|
||||
@@ -19,6 +32,21 @@ pub(crate) struct BarkPushConfig {
|
||||
pub(crate) template: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BarkPushConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BarkPushConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field(
|
||||
"device_key",
|
||||
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("server_url", &self.server_url)
|
||||
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||
let value = state
|
||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||
@@ -35,23 +63,34 @@ pub(crate) async fn read_bark_push_config(
|
||||
state: &AppState,
|
||||
) -> Result<BarkPushConfig, GatewayError> {
|
||||
let enabled = bark_push_module_enabled(state).await?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let server_url = state
|
||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let server_url = normalized_bark_server_url(&server_url)?;
|
||||
let binding = bark_device_key_binding(&server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
let device_key = match device_key {
|
||||
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(device_key) = device_key.as_deref() {
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
}
|
||||
let template = state
|
||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
if let Some(template) = template.as_deref() {
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
}
|
||||
|
||||
Ok(BarkPushConfig {
|
||||
enabled,
|
||||
@@ -62,7 +101,7 @@ pub(crate) async fn read_bark_push_config(
|
||||
}
|
||||
|
||||
pub(crate) async fn send_bark_push(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
config: &BarkPushConfig,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
@@ -76,11 +115,13 @@ pub(crate) async fn send_bark_push(
|
||||
"Bark Device Key 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
||||
let response = state
|
||||
.client
|
||||
.post(format!("{server_url}/push"))
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||
let response = client
|
||||
.post(push_url)
|
||||
.json(&json!({
|
||||
"device_key": device_key,
|
||||
"title": title,
|
||||
@@ -88,16 +129,14 @@ pub(crate) async fn send_bark_push(
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||
let status = response.status();
|
||||
let text = response
|
||||
.text()
|
||||
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
if !status.is_success() {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 返回 HTTP {status}: {text}"
|
||||
)));
|
||||
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||
}
|
||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||
let code_is_ok = payload
|
||||
@@ -114,53 +153,260 @@ pub(crate) async fn send_bark_push(
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if !code_is_ok {
|
||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
||||
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
let server_url = server_url.trim().trim_end_matches('/');
|
||||
if server_url.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(server_url.to_string())
|
||||
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
|
||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
||||
match template {
|
||||
Some(template) if !template.trim().is_empty() => template
|
||||
.replace("{title}", title)
|
||||
.replace("{body}", markdown_body),
|
||||
_ => markdown_body.to_string(),
|
||||
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||
match error {
|
||||
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||
format!("Bark 响应超过 {max_bytes} 字节")
|
||||
}
|
||||
aether_http::ResponseBodyReadError::Read(error) => {
|
||||
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else {
|
||||
"transport"
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
canonical_bark_server_url(server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||
}
|
||||
|
||||
async fn build_bark_push_client_and_url(
|
||||
server_url: &str,
|
||||
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let normalized = normalized_bark_server_url(server_url)?;
|
||||
let mut push_url = url::Url::parse(&normalized)
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||
|
||||
let host = push_url
|
||||
.host_str()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||
.to_string();
|
||||
let port = push_url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||
let addresses = aether_http::lookup_host_with_limits(
|
||||
host.as_str(),
|
||||
port,
|
||||
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error.kind() {
|
||||
std::io::ErrorKind::TimedOut => "Bark 服务器 DNS 解析超时",
|
||||
std::io::ErrorKind::InvalidData => "Bark 服务器 DNS 解析返回过多地址",
|
||||
_ => "Bark 服务器 DNS 解析失败",
|
||||
};
|
||||
GatewayError::Internal(message.to_string())
|
||||
})?;
|
||||
let allow_benchmarking_ip = push_url.scheme() == "https"
|
||||
&& push_url.port_or_known_default() == Some(443)
|
||||
&& host.eq_ignore_ascii_case("api.day.app");
|
||||
validate_bark_resolved_addresses(
|
||||
&addresses,
|
||||
env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV),
|
||||
allow_benchmarking_ip,
|
||||
)?;
|
||||
|
||||
push_url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||
.pop_if_empty()
|
||||
.push("push");
|
||||
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||
}
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||
Ok((client, push_url))
|
||||
}
|
||||
|
||||
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||
if url.scheme() == "http" && !allow_http {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_resolved_addresses(
|
||||
addresses: &[SocketAddr],
|
||||
allow_private: bool,
|
||||
allow_benchmarking_ip: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||
));
|
||||
}
|
||||
if !allow_private
|
||||
&& addresses.iter().any(|address| {
|
||||
aether_http::is_private_or_reserved_ip(address.ip())
|
||||
&& !(allow_benchmarking_ip
|
||||
&& aether_http::is_ipv4_benchmarking_fake_ip(address.ip()))
|
||||
})
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn env_flag_enabled(key: &str) -> bool {
|
||||
std::env::var(key).ok().is_some_and(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_bark_config_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_content_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_bark_config_field(field, value, max_bytes)
|
||||
}
|
||||
|
||||
fn render_bark_body(
|
||||
template: Option<&str>,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let template = template
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or("{body}");
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
|
||||
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||
let mut cursor = 0usize;
|
||||
while cursor < template.len() {
|
||||
let remaining = &template[cursor..];
|
||||
let title_match = remaining.find("{title}");
|
||||
let body_match = remaining.find("{body}");
|
||||
let next = match (title_match, body_match) {
|
||||
(None, None) => {
|
||||
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||
cursor = template.len();
|
||||
continue;
|
||||
}
|
||||
(Some(index), None) => (index, "{title}", title),
|
||||
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||
(title_index, "{title}", title)
|
||||
}
|
||||
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||
};
|
||||
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||
cursor += next.0 + next.1.len();
|
||||
}
|
||||
if rendered.is_empty() && template.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
Ok(rendered)
|
||||
}
|
||||
|
||||
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||
let next_len = output
|
||||
.len()
|
||||
.checked_add(part.len())
|
||||
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark rendered body exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
output.push_str(part);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{normalized_bark_server_url, render_bark_body};
|
||||
use super::{
|
||||
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||
};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
#[test]
|
||||
fn bark_body_uses_template_when_provided() {
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||
.expect("template should render");
|
||||
assert_eq!(rendered, "告警\n\n原始正文");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
||||
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -170,4 +416,61 @@ mod tests {
|
||||
"https://api.day.app"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||
for invalid in [
|
||||
"https://[email protected]",
|
||||
"https://example.com?target=internal",
|
||||
"https://example.com/#fragment",
|
||||
] {
|
||||
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_http_transport_requires_explicit_opt_in() {
|
||||
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_private_targets_require_explicit_opt_in() {
|
||||
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&private, false, false).is_err());
|
||||
assert!(validate_bark_resolved_addresses(&private, true, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_builtin_server_allows_benchmarking_ip_only_with_https_default_port() {
|
||||
let fake = [SocketAddr::from(([198, 18, 75, 234], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, true).is_ok());
|
||||
assert!(validate_bark_resolved_addresses(
|
||||
&[fake[0], SocketAddr::from(([127, 0, 0, 1], 443))],
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.is_err());
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, false).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||
let secret = "bark-secret-query";
|
||||
let error = reqwest::Client::new()
|
||||
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||
|
||||
let message = bark_request_error_message(&error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
|
||||
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||
let message = bark_response_body_error_message(&body_error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@
|
||||
#[path = "support/responses_ws_probe.rs"]
|
||||
mod responses_ws_probe;
|
||||
|
||||
use aether_gateway::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
use aether_gateway::{codex_client_originator, codex_client_user_agent};
|
||||
use clap::Parser;
|
||||
use http::header::{AUTHORIZATION, USER_AGENT};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
@@ -78,14 +78,12 @@ fn handshake_headers(access_token: &str, account_id: &str) -> Result<HeaderMap,
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(AUTHORIZATION, bearer_authorization_value(access_token)?);
|
||||
headers.insert(HeaderName::from_static("chatgpt-account-id"), account_id);
|
||||
headers.insert(
|
||||
USER_AGENT,
|
||||
HeaderValue::from_static(CODEX_CLIENT_USER_AGENT),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("originator"),
|
||||
HeaderValue::from_static(CODEX_CLIENT_ORIGINATOR),
|
||||
);
|
||||
let user_agent = HeaderValue::from_str(&codex_client_user_agent())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(USER_AGENT, user_agent);
|
||||
let originator = HeaderValue::from_str(&codex_client_originator())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(HeaderName::from_static("originator"), originator);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -111,6 +109,18 @@ mod tests {
|
||||
assert!(headers.contains_key("chatgpt-account-id"));
|
||||
assert!(headers.contains_key(USER_AGENT));
|
||||
assert!(headers.contains_key("originator"));
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(USER_AGENT)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("originator")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_originator().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
CodexResponsesProbeProfile::sent_header_names(),
|
||||
vec![
|
||||
|
||||
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||
{
|
||||
return Err(ProbeFailure::InvalidEndpoint);
|
||||
}
|
||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
||||
|
||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||
let client = wreq::Client::builder()
|
||||
.no_proxy()
|
||||
.connect_timeout(config.turn_timeout)
|
||||
.timeout(config.turn_timeout)
|
||||
.build()
|
||||
@@ -434,7 +436,14 @@ mod tests {
|
||||
#[test]
|
||||
fn probe_url_rejects_credentials_and_query_strings() {
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||
}
|
||||
|
||||
@@ -422,6 +422,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate(&self, key: &str) {
|
||||
let Ok(_mutation) = self.mutation.lock() else {
|
||||
return;
|
||||
};
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
self.entries.remove(&key.to_string());
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn insert_if_generation(
|
||||
&self,
|
||||
key: String,
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
//! Codex 客户端画像的运行时发布与官方 CLI 版本刷新。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_runtime_state::RuntimeState;
|
||||
use futures_util::StreamExt as _;
|
||||
use reqwest::{redirect::Policy, Client};
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::ai_serving::api::{codex_client_version, set_codex_cli_version};
|
||||
use crate::AppState;
|
||||
|
||||
const CLI_RELEASE_ENDPOINT: &str = "https://registry.npmjs.org/@openai%2Fcodex/latest";
|
||||
const PROFILE_CACHE_KEY: &str = "aether:codex:client-profile:v1";
|
||||
const PROFILE_CACHE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
|
||||
const PROFILE_REFRESH_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const RELEASE_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const RELEASE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_RELEASE_BYTES: usize = 256 * 1024;
|
||||
const CLI_TARGETS: [&str; 6] = [
|
||||
"darwin-arm64",
|
||||
"darwin-x64",
|
||||
"linux-arm64",
|
||||
"linux-x64",
|
||||
"win32-arm64",
|
||||
"win32-x64",
|
||||
];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct NpmRelease {
|
||||
name: String,
|
||||
version: String,
|
||||
optional_dependencies: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
struct CachedProfile {
|
||||
version: String,
|
||||
verified_at_unix_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum ProfileRefreshError {
|
||||
#[error("Codex CLI release client initialization failed: {0}")]
|
||||
Client(#[from] reqwest::Error),
|
||||
#[error("Codex CLI release request returned HTTP {0}")]
|
||||
HttpStatus(u16),
|
||||
#[error("Codex CLI release response exceeded {MAX_RELEASE_BYTES} bytes")]
|
||||
ResponseTooLarge,
|
||||
#[error("Codex CLI release metadata is invalid")]
|
||||
InvalidMetadata,
|
||||
#[error("Codex CLI release version is older than the active profile")]
|
||||
Rollback,
|
||||
#[error("Codex CLI profile cache operation failed: {0}")]
|
||||
Cache(String),
|
||||
}
|
||||
|
||||
fn version_sequence(version: &str) -> Result<u64, ProfileRefreshError> {
|
||||
let parsed = Version::parse(version).map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
if !parsed.pre.is_empty()
|
||||
|| !parsed.build.is_empty()
|
||||
|| parsed.major > 999
|
||||
|| parsed.minor > 999
|
||||
|| parsed.patch > 999
|
||||
{
|
||||
return Err(ProfileRefreshError::InvalidMetadata);
|
||||
}
|
||||
Ok(1 + parsed.major * 1_000_000 + parsed.minor * 1_000 + parsed.patch)
|
||||
}
|
||||
|
||||
/// 校验官方 npm stable 标签及六个平台依赖来自同一版本发布。
|
||||
fn parse_cli_release(bytes: &[u8]) -> Result<String, ProfileRefreshError> {
|
||||
if bytes.len() > MAX_RELEASE_BYTES {
|
||||
return Err(ProfileRefreshError::ResponseTooLarge);
|
||||
}
|
||||
let release = serde_json::from_slice::<NpmRelease>(bytes)
|
||||
.map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
let sequence = version_sequence(&release.version)?;
|
||||
if sequence == 0
|
||||
|| release.name != "@openai/codex"
|
||||
|| CLI_TARGETS.iter().any(|target| {
|
||||
release
|
||||
.optional_dependencies
|
||||
.get(&format!("@openai/codex-{target}"))
|
||||
!= Some(&format!("npm:@openai/codex@{}-{target}", release.version))
|
||||
})
|
||||
{
|
||||
return Err(ProfileRefreshError::InvalidMetadata);
|
||||
}
|
||||
Ok(release.version)
|
||||
}
|
||||
|
||||
fn refresh_enabled_from(value: Option<&str>) -> bool {
|
||||
!value.is_some_and(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"0" | "false" | "off"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn refresh_enabled() -> bool {
|
||||
refresh_enabled_from(
|
||||
std::env::var("AETHER_CODEX_CLIENT_PROFILE_REFRESH")
|
||||
.ok()
|
||||
.as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
fn fixed_version_from(value: Option<&str>) -> Option<String> {
|
||||
let value = value?.trim();
|
||||
if value.is_empty() || version_sequence(value).is_err() {
|
||||
None
|
||||
} else {
|
||||
Some(value.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed_version_override() -> Option<String> {
|
||||
let value = std::env::var("AETHER_CODEX_CLIENT_VERSION").ok()?;
|
||||
let version = fixed_version_from(Some(&value));
|
||||
if version.is_none() {
|
||||
warn!(
|
||||
event_name = "codex_client_profile_fixed_version_invalid",
|
||||
"AETHER_CODEX_CLIENT_VERSION is invalid; using cached or built-in profile"
|
||||
);
|
||||
}
|
||||
version
|
||||
}
|
||||
|
||||
fn build_release_client() -> Result<Client, ProfileRefreshError> {
|
||||
Client::builder()
|
||||
.https_only(true)
|
||||
.no_proxy()
|
||||
.redirect(Policy::none())
|
||||
.connect_timeout(RELEASE_CONNECT_TIMEOUT)
|
||||
.timeout(RELEASE_REQUEST_TIMEOUT)
|
||||
.build()
|
||||
.map_err(ProfileRefreshError::Client)
|
||||
}
|
||||
|
||||
async fn fetch_latest_cli_version(client: &Client) -> Result<String, ProfileRefreshError> {
|
||||
let response = client
|
||||
.get(CLI_RELEASE_ENDPOINT)
|
||||
.send()
|
||||
.await
|
||||
.map_err(ProfileRefreshError::Client)?;
|
||||
if !response.status().is_success() {
|
||||
return Err(ProfileRefreshError::HttpStatus(response.status().as_u16()));
|
||||
}
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > MAX_RELEASE_BYTES as u64)
|
||||
{
|
||||
return Err(ProfileRefreshError::ResponseTooLarge);
|
||||
}
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(ProfileRefreshError::Client)?;
|
||||
if bytes.len().saturating_add(chunk.len()) > MAX_RELEASE_BYTES {
|
||||
return Err(ProfileRefreshError::ResponseTooLarge);
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
parse_cli_release(&bytes)
|
||||
}
|
||||
|
||||
async fn restore_cached_profile(runtime: &RuntimeState) -> Result<(), ProfileRefreshError> {
|
||||
let Some(raw) = runtime
|
||||
.kv_get(PROFILE_CACHE_KEY)
|
||||
.await
|
||||
.map_err(|err| ProfileRefreshError::Cache(err.to_string()))?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let cached = serde_json::from_str::<CachedProfile>(&raw)
|
||||
.map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
if let Some(version) = cached_version_to_restore(&cached, &codex_client_version())? {
|
||||
set_codex_cli_version(&version).map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
info!(
|
||||
event_name = "codex_client_profile_restored",
|
||||
version = %version,
|
||||
verified_at_unix_secs = cached.verified_at_unix_secs,
|
||||
"restored cached Codex CLI profile"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cached_version_to_restore(
|
||||
cached: &CachedProfile,
|
||||
active_version: &str,
|
||||
) -> Result<Option<String>, ProfileRefreshError> {
|
||||
let cached_sequence = version_sequence(&cached.version)?;
|
||||
let active_sequence = version_sequence(active_version)?;
|
||||
Ok((cached_sequence >= active_sequence).then(|| cached.version.clone()))
|
||||
}
|
||||
|
||||
async fn refresh_once_with_fetch<F, Fut>(
|
||||
runtime: &RuntimeState,
|
||||
fixed_version: Option<&str>,
|
||||
refresh_is_enabled: bool,
|
||||
fetch_latest: F,
|
||||
) -> Result<String, ProfileRefreshError>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<String, ProfileRefreshError>>,
|
||||
{
|
||||
if let Some(version) = fixed_version {
|
||||
set_codex_cli_version(version).map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
return Ok(version.to_owned());
|
||||
}
|
||||
|
||||
if let Err(error) = restore_cached_profile(runtime).await {
|
||||
// 缓存损坏或暂时不可用不应阻断官方版本检查;当前进程继续使用旧画像。
|
||||
warn!(
|
||||
event_name = "codex_client_profile_cache_restore_failed",
|
||||
error = %error,
|
||||
"could not restore cached Codex CLI profile"
|
||||
);
|
||||
}
|
||||
if !refresh_is_enabled {
|
||||
return Ok(codex_client_version());
|
||||
}
|
||||
|
||||
let version = fetch_latest().await?;
|
||||
let current = codex_client_version();
|
||||
if version_sequence(&version)? < version_sequence(¤t)? {
|
||||
return Err(ProfileRefreshError::Rollback);
|
||||
}
|
||||
|
||||
let cached = CachedProfile {
|
||||
version: version.clone(),
|
||||
verified_at_unix_secs: chrono::Utc::now().timestamp().max(0) as u64,
|
||||
};
|
||||
let serialized =
|
||||
serde_json::to_string(&cached).map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
set_codex_cli_version(&version).map_err(|_| ProfileRefreshError::InvalidMetadata)?;
|
||||
if let Err(error) = runtime
|
||||
.kv_set(PROFILE_CACHE_KEY, serialized, Some(PROFILE_CACHE_TTL))
|
||||
.await
|
||||
{
|
||||
// 本地画像已经完成原子替换;缓存写失败只影响下次进程启动的恢复。
|
||||
warn!(
|
||||
event_name = "codex_client_profile_cache_write_failed",
|
||||
error = %error,
|
||||
"published Codex CLI profile locally but could not persist the cache"
|
||||
);
|
||||
}
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
async fn refresh_once(runtime: &RuntimeState) -> Result<String, ProfileRefreshError> {
|
||||
let fixed_version = fixed_version_override();
|
||||
refresh_once_with_fetch(
|
||||
runtime,
|
||||
fixed_version.as_deref(),
|
||||
refresh_enabled(),
|
||||
|| async {
|
||||
let client = build_release_client()?;
|
||||
fetch_latest_cli_version(&client).await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn prewarm(runtime: &RuntimeState) -> Result<String, String> {
|
||||
refresh_once(runtime).await.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_worker(app: AppState) -> tokio::task::JoinHandle<()> {
|
||||
crate::task_runtime::spawn_singleton_worker(
|
||||
app,
|
||||
crate::task_runtime::TASK_KEY_CODEX_CLIENT_PROFILE,
|
||||
|app| async move {
|
||||
let mut interval = tokio::time::interval(PROFILE_REFRESH_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
// 启动阶段由 prewarm 完成一次检查;后台任务只负责后续每日刷新,避免重复建连。
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
match refresh_once(app.runtime_state()).await {
|
||||
Ok(version) => info!(
|
||||
event_name = "codex_client_profile_refreshed",
|
||||
version = %version,
|
||||
"refreshed Codex CLI profile"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
event_name = "codex_client_profile_refresh_failed",
|
||||
error = %error,
|
||||
"keeping the previous Codex CLI profile after refresh failure"
|
||||
),
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Mutex, OnceLock,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||
|
||||
use super::{
|
||||
cached_version_to_restore, fixed_version_from, parse_cli_release, refresh_enabled_from,
|
||||
refresh_once_with_fetch, CachedProfile, ProfileRefreshError, PROFILE_CACHE_KEY,
|
||||
};
|
||||
use crate::ai_serving::api::{
|
||||
codex_client_profile, codex_client_version, set_codex_cli_version,
|
||||
set_codex_client_profile, CodexClientProfile,
|
||||
};
|
||||
|
||||
static PROFILE_TEST_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
|
||||
struct ProfileRestore(CodexClientProfile);
|
||||
|
||||
impl Drop for ProfileRestore {
|
||||
fn drop(&mut self) {
|
||||
set_codex_client_profile(self.0.clone());
|
||||
}
|
||||
}
|
||||
|
||||
fn profile_restore_guard() -> (std::sync::MutexGuard<'static, ()>, ProfileRestore) {
|
||||
let lock = PROFILE_TEST_LOCK.get_or_init(|| Mutex::new(()));
|
||||
let guard = lock.lock().expect("profile test lock");
|
||||
let restore = ProfileRestore(codex_client_profile());
|
||||
(guard, restore)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_only_one_verified_cli_release_for_all_targets() {
|
||||
let body = serde_json::json!({
|
||||
"name": "@openai/codex",
|
||||
"version": "0.200.1",
|
||||
"optionalDependencies": {
|
||||
"@openai/codex-darwin-arm64": "npm:@openai/[email protected]",
|
||||
"@openai/codex-darwin-x64": "npm:@openai/[email protected]",
|
||||
"@openai/codex-linux-arm64": "npm:@openai/[email protected]",
|
||||
"@openai/codex-linux-x64": "npm:@openai/[email protected]",
|
||||
"@openai/codex-win32-arm64": "npm:@openai/[email protected]",
|
||||
"@openai/codex-win32-x64": "npm:@openai/[email protected]"
|
||||
}
|
||||
});
|
||||
assert_eq!(
|
||||
parse_cli_release(&serde_json::to_vec(&body).unwrap()).unwrap(),
|
||||
"0.200.1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_incomplete_platform_release() {
|
||||
let body = serde_json::json!({
|
||||
"name": "@openai/codex",
|
||||
"version": "0.200.1",
|
||||
"optionalDependencies": {}
|
||||
});
|
||||
assert!(parse_cli_release(&serde_json::to_vec(&body).unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_and_fixed_version_environment_policies_are_strict() {
|
||||
assert!(!refresh_enabled_from(Some("off")));
|
||||
assert!(!refresh_enabled_from(Some(" FALSE ")));
|
||||
assert!(refresh_enabled_from(None));
|
||||
assert_eq!(
|
||||
fixed_version_from(Some(" 0.200.1 ")).as_deref(),
|
||||
Some("0.200.1")
|
||||
);
|
||||
assert!(fixed_version_from(Some("0.200.1-beta.1")).is_none());
|
||||
assert!(fixed_version_from(Some("1.2")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_profile_never_rewinds_active_profile() {
|
||||
let cached = CachedProfile {
|
||||
version: "0.200.1".to_string(),
|
||||
verified_at_unix_secs: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
cached_version_to_restore(&cached, "0.200.0").unwrap(),
|
||||
Some("0.200.1".to_string())
|
||||
);
|
||||
assert_eq!(cached_version_to_restore(&cached, "0.201.0").unwrap(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_hit_is_restored_without_network_when_refresh_is_disabled() {
|
||||
let (_lock, _restore) = profile_restore_guard();
|
||||
let runtime = RuntimeState::memory(MemoryRuntimeStateConfig::default());
|
||||
runtime
|
||||
.kv_set(
|
||||
PROFILE_CACHE_KEY,
|
||||
serde_json::to_string(&CachedProfile {
|
||||
version: "0.200.1".to_string(),
|
||||
verified_at_unix_secs: 1,
|
||||
})
|
||||
.unwrap(),
|
||||
Some(Duration::from_secs(60)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = refresh_once_with_fetch(&runtime, None, false, || async {
|
||||
Err(ProfileRefreshError::HttpStatus(599))
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, "0.200.1");
|
||||
assert_eq!(codex_client_version(), "0.200.1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_failure_keeps_previous_profile() {
|
||||
let (_lock, _restore) = profile_restore_guard();
|
||||
let runtime = RuntimeState::memory(MemoryRuntimeStateConfig::default());
|
||||
let before = codex_client_profile();
|
||||
let result = refresh_once_with_fetch(&runtime, None, true, || async {
|
||||
Err(ProfileRefreshError::HttpStatus(503))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProfileRefreshError::HttpStatus(503))));
|
||||
assert_eq!(codex_client_profile(), before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fixed_version_override_skips_network_and_publishes_profile() {
|
||||
let (_lock, _restore) = profile_restore_guard();
|
||||
let runtime = RuntimeState::memory(MemoryRuntimeStateConfig::default());
|
||||
let fetch_called = AtomicBool::new(false);
|
||||
let result = refresh_once_with_fetch(&runtime, Some("0.220.0"), true, || async {
|
||||
fetch_called.store(true, Ordering::SeqCst);
|
||||
Ok("0.221.0".to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, "0.220.0");
|
||||
assert!(!fetch_called.load(Ordering::SeqCst));
|
||||
assert_eq!(codex_client_version(), "0.220.0");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_is_rejected_without_replacing_profile() {
|
||||
let (_lock, _restore) = profile_restore_guard();
|
||||
set_codex_cli_version("0.220.0").unwrap();
|
||||
let runtime = RuntimeState::memory(MemoryRuntimeStateConfig::default());
|
||||
let result =
|
||||
refresh_once_with_fetch(&runtime, None, true, || async { Ok("0.219.9".to_string()) })
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ProfileRefreshError::Rollback)));
|
||||
assert_eq!(codex_client_version(), "0.220.0");
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-forwarded-by";
|
||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-owner-instance-id";
|
||||
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||
"local_proxy_passthrough_removed";
|
||||
@@ -139,6 +140,8 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/models/{model}/operations/{id}",
|
||||
"/v1beta/operations",
|
||||
"/v1beta/operations/{id}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{path...}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{path...}",
|
||||
"/upload/v1beta/files",
|
||||
|
||||
@@ -23,7 +23,7 @@ pub(crate) fn extract_requested_model(
|
||||
body: &Bytes,
|
||||
) -> Option<String> {
|
||||
if decision.route_family.as_deref() == Some("gemini") {
|
||||
if let Some(model) = extract_gemini_model_from_path(uri.path()) {
|
||||
if let Some(model) = extract_gemini_requested_model_from_path(uri.path()) {
|
||||
return Some(model);
|
||||
}
|
||||
}
|
||||
@@ -43,23 +43,46 @@ pub(crate) fn extract_requested_model(
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn extract_gemini_requested_model_from_path(path: &str) -> Option<String> {
|
||||
let model = extract_gemini_model_from_path(path)?;
|
||||
Some(
|
||||
model
|
||||
.split_once("/operations/")
|
||||
.map(|(model, _)| model)
|
||||
.unwrap_or(model.as_str())
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> GatewayExtractedCredentials {
|
||||
extract_request_credentials_with_trusted_auth(headers, uri, auth_endpoint_signature, cfg!(test))
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> GatewayExtractedCredentials {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: header_value_str(headers, http::header::AUTHORIZATION.as_str())
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
authorization_bearer: unique_header_value_str(
|
||||
headers,
|
||||
http::header::AUTHORIZATION.as_str(),
|
||||
)
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
x_api_key: header_value_str(headers, "x-api-key"),
|
||||
api_key: header_value_str(headers, "api-key"),
|
||||
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
||||
query_key: extract_query_api_key(uri),
|
||||
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
||||
};
|
||||
let trusted_headers = extract_trusted_auth_headers(headers);
|
||||
let trusted_headers = extract_trusted_auth_headers(headers, trusted_auth_verified);
|
||||
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
||||
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
||||
|
||||
@@ -71,6 +94,20 @@ pub(super) fn extract_request_credentials(
|
||||
}
|
||||
}
|
||||
|
||||
fn unique_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<String> {
|
||||
let mut values = headers.get_all(key).iter();
|
||||
let value = values.next()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
@@ -85,6 +122,7 @@ pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
||||
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
@@ -97,13 +135,32 @@ pub(super) fn build_auth_context_cache_key(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> Option<String> {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn build_auth_context_cache_key_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<String> {
|
||||
let signature = auth_endpoint_signature.trim();
|
||||
if signature.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let trusted_headers = extracted.trusted_headers;
|
||||
let bundle = extracted.bundle;
|
||||
if bundle.authorization_bearer.is_none()
|
||||
@@ -135,7 +192,7 @@ pub(super) fn build_auth_context_cache_key(
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Some(format!(
|
||||
let raw_cache_identity = format!(
|
||||
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||
bundle.authorization_bearer.unwrap_or_default(),
|
||||
bundle.x_api_key.unwrap_or_default(),
|
||||
@@ -147,11 +204,26 @@ pub(super) fn build_auth_context_cache_key(
|
||||
trusted_api_key_id,
|
||||
trusted_balance_remaining,
|
||||
trusted_access_allowed,
|
||||
))
|
||||
);
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(raw_cache_identity.as_bytes());
|
||||
Some(format!("auth-context:sha256:{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
fn extract_trusted_auth_headers(headers: &http::HeaderMap) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !has_trusted_gateway_marker(headers) {
|
||||
fn extract_trusted_auth_headers(
|
||||
headers: &http::HeaderMap,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !trusted_auth_verified {
|
||||
return None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.starts_with("rust-phase3")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
||||
@@ -387,7 +459,7 @@ fn extract_bearer_token(value: &str) -> Option<&str> {
|
||||
return None;
|
||||
}
|
||||
let token = token.trim();
|
||||
if token.is_empty() {
|
||||
if token.is_empty() || token.chars().any(char::is_whitespace) {
|
||||
None
|
||||
} else {
|
||||
Some(token)
|
||||
@@ -472,6 +544,46 @@ mod tests {
|
||||
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_requested_model_handles_gemini_generation_and_operation_paths() {
|
||||
let generation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/gemini-2.5-pro:generateContent",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("generate_content".to_string()),
|
||||
Some("gemini:generate_content".to_string()),
|
||||
);
|
||||
let operation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/veo-3/operations/task-123:cancel",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("video".to_string()),
|
||||
Some("gemini:video".to_string()),
|
||||
);
|
||||
let headers = http::HeaderMap::new();
|
||||
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&generation_decision,
|
||||
&uri("/v1beta/models/gemini-2.5-pro:generateContent"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("gemini-2.5-pro")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&operation_decision,
|
||||
&uri("/v1beta/models/veo-3/operations/task-123:cancel"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("veo-3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_openai_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -491,6 +603,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_or_combined_authorization_credentials() {
|
||||
let mut duplicate = http::HeaderMap::new();
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token".parse().unwrap(),
|
||||
);
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&duplicate, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
|
||||
let mut combined = http::HeaderMap::new();
|
||||
combined.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token, Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&combined, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_codex_live_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -608,7 +747,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_key_includes_cookie_header() {
|
||||
fn cache_key_hashes_cookie_header_instead_of_retaining_session_secret() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
||||
|
||||
@@ -618,7 +757,8 @@ mod tests {
|
||||
"internal:session",
|
||||
)
|
||||
.expect("cache key should exist");
|
||||
assert!(cache_key.contains("session=abc123"));
|
||||
assert!(cache_key.starts_with("auth-context:sha256:"));
|
||||
assert!(!cache_key.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -669,12 +809,10 @@ mod tests {
|
||||
.expect("trusted cache key should exist");
|
||||
|
||||
assert_ne!(first, second);
|
||||
assert!(first.contains("user-1"));
|
||||
assert!(first.contains("key-1"));
|
||||
assert!(first.contains("1.5"));
|
||||
assert!(first.contains("true"));
|
||||
assert!(second.contains("user-2"));
|
||||
assert!(second.contains("false"));
|
||||
for raw_identity in ["user-1", "key-1", "1.5", "user-2"] {
|
||||
assert!(!first.contains(raw_identity));
|
||||
assert!(!second.contains(raw_identity));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -224,7 +224,7 @@ fn wallet_finite_available_usd(
|
||||
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
||||
}
|
||||
|
||||
async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
state: &AppState,
|
||||
plan: &aether_contracts::ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
@@ -925,6 +925,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: Some(allowed_models),
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
});
|
||||
decision
|
||||
}
|
||||
|
||||
@@ -7,13 +7,16 @@ mod types;
|
||||
pub(crate) use credentials::extract_requested_model;
|
||||
pub(super) use credentials::resolve_gateway_credential_carrier;
|
||||
pub(crate) use gate::{
|
||||
execution_plan_balance_capacity_rejection, request_model_local_rejection,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
request_model_local_rejection, should_buffer_request_for_local_auth,
|
||||
trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
};
|
||||
pub(crate) use resolution::{
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
resolve_execution_runtime_auth_context, GatewayAdminPrincipalContext,
|
||||
GatewayControlAuthContext,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext,
|
||||
};
|
||||
pub(super) use resolution::{
|
||||
resolve_control_decision_auth_with_trusted_auth, ControlDecisionAuthResolution,
|
||||
};
|
||||
pub(super) use resolution::{resolve_control_decision_auth, ControlDecisionAuthResolution};
|
||||
pub(crate) use types::GatewayCredentialCarrier;
|
||||
|
||||
@@ -4,11 +4,9 @@ use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
||||
};
|
||||
use axum::http::Uri;
|
||||
use base64::Engine as _;
|
||||
use hmac::Mac;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::wallet_runtime::{
|
||||
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
||||
@@ -17,7 +15,8 @@ use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::GatewayControlDecision;
|
||||
use super::credentials::{
|
||||
build_auth_context_cache_key, current_unix_secs, extract_request_credentials,
|
||||
build_auth_context_cache_key, build_auth_context_cache_key_with_trusted_auth,
|
||||
current_unix_secs, extract_request_credentials, extract_request_credentials_with_trusted_auth,
|
||||
extract_trusted_admin_headers, hash_api_key,
|
||||
};
|
||||
use super::gate::GatewayLocalAuthRejection;
|
||||
@@ -27,6 +26,9 @@ use super::types::{
|
||||
};
|
||||
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
||||
use crate::headers::header_value_str;
|
||||
use crate::local_auth_token::{
|
||||
decode_local_auth_token, local_auth_token_identity_matches_user, LocalAuthTokenType,
|
||||
};
|
||||
|
||||
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
||||
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
||||
@@ -93,6 +95,30 @@ pub(crate) struct GatewayControlAuthContext {
|
||||
pub(crate) allowed_models: Option<Vec<String>>,
|
||||
#[serde(skip)]
|
||||
pub(crate) ip_rules: Option<Vec<String>>,
|
||||
/// Credential verifier that established this API-key identity. Long-lived
|
||||
/// executions use it to prove that a later row with the same IDs is still
|
||||
/// the record authenticated by the original request.
|
||||
#[serde(skip)]
|
||||
pub(crate) verified_api_key_hash: Option<VerifiedApiKeyHash>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct VerifiedApiKeyHash(String);
|
||||
|
||||
impl VerifiedApiKeyHash {
|
||||
fn new(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
fn as_str(&self) -> &str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for VerifiedApiKeyHash {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("VerifiedApiKeyHash([REDACTED])")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -109,11 +135,30 @@ pub(in super::super) enum ControlDecisionAuthResolution {
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
decision: GatewayControlDecision,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
resolve_control_decision_auth_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
trace_id,
|
||||
decision,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
mut decision: GatewayControlDecision,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
if let Some(admin_principal) =
|
||||
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
||||
@@ -132,10 +177,18 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.admin_principal = Some(admin_principal);
|
||||
}
|
||||
|
||||
let auth_context_cache_key = decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature));
|
||||
let auth_context_cache_key =
|
||||
decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
});
|
||||
|
||||
let mut resolved_auth_context = None;
|
||||
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
||||
@@ -149,6 +202,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
headers,
|
||||
uri,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
@@ -168,6 +222,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
uri,
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
true,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -336,7 +391,7 @@ async fn resolve_local_admin_principal(
|
||||
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let claims = match decode_local_auth_token(access_token, "access") {
|
||||
let claims = match decode_local_auth_token(access_token, LocalAuthTokenType::Access) {
|
||||
Ok(claims) => claims,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
@@ -351,6 +406,14 @@ async fn resolve_local_admin_principal(
|
||||
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_local_admin_session_principal(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
) -> Result<Option<GatewayAdminPrincipalContext>, GatewayError> {
|
||||
resolve_local_admin_principal(state, headers, uri, Some("admin:operational")).await
|
||||
}
|
||||
|
||||
async fn resolve_local_admin_principal_from_claims(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
@@ -373,6 +436,9 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !local_auth_token_identity_matches_user(claims, &user) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let now = chrono::Utc::now();
|
||||
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
||||
@@ -380,6 +446,7 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
};
|
||||
if session.is_revoked()
|
||||
|| session.is_expired(now)
|
||||
|| session.security_version != user.security_version
|
||||
|| session.client_device_id != client_device_id
|
||||
{
|
||||
return Ok(None);
|
||||
@@ -431,68 +498,6 @@ fn local_admin_user_agent(headers: &http::HeaderMap) -> Option<String> {
|
||||
.map(|value| value.chars().take(1000).collect())
|
||||
}
|
||||
|
||||
fn local_auth_secret() -> String {
|
||||
std::env::var("JWT_SECRET_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| "aether-rust-dev-jwt-secret".to_string())
|
||||
}
|
||||
|
||||
fn decode_local_auth_token(
|
||||
token: &str,
|
||||
expected_type: &str,
|
||||
) -> Result<serde_json::Map<String, Value>, String> {
|
||||
let mut parts = token.split('.');
|
||||
let Some(header_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(payload_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(signature_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
if parts.next().is_some() {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
|
||||
let signing_input = format!("{header_segment}.{payload_segment}");
|
||||
let signature = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(signature_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let mut mac = hmac::Hmac::<sha2::Sha256>::new_from_slice(local_auth_secret().as_bytes())
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
mac.update(signing_input.as_bytes());
|
||||
mac.verify_slice(&signature)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
|
||||
let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(payload_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let payload =
|
||||
serde_json::from_slice::<Value>(&payload_bytes).map_err(|_| "invalid token".to_string())?;
|
||||
let payload = payload
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
let actual_type = payload
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if actual_type != expected_type {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
let exp = payload
|
||||
.get("exp")
|
||||
.and_then(Value::as_i64)
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
if exp <= chrono::Utc::now().timestamp() {
|
||||
return Err("expired token".to_string());
|
||||
}
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
state: &AppState,
|
||||
decision: &GatewayControlDecision,
|
||||
@@ -525,6 +530,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
Some(auth_endpoint_signature),
|
||||
headers,
|
||||
uri,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
.map(Some);
|
||||
@@ -539,6 +545,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
uri,
|
||||
Some(auth_endpoint_signature),
|
||||
true,
|
||||
cfg!(test),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -558,6 +565,7 @@ async fn revalidate_cached_auth_context(
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if is_negative_auth_context(&auth_context)
|
||||
|| !auth_context.access_allowed
|
||||
@@ -581,6 +589,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context.clone(),
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -622,6 +631,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
if refreshed.is_err() {
|
||||
@@ -639,9 +649,16 @@ async fn resolve_security_fresh_auth_context(
|
||||
uri: &Uri,
|
||||
stale: GatewayControlAuthContext,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if let Some(refreshed) =
|
||||
resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature).await?
|
||||
if let Some(refreshed) = resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(refreshed);
|
||||
}
|
||||
@@ -660,19 +677,27 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
cache_negative: bool,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(cache_key) = cache_key else {
|
||||
return resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature)
|
||||
.await;
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
loop {
|
||||
match state.auth_context_cache.register_inflight(cache_key) {
|
||||
AuthContextInflightRegistration::Leader(guard) => {
|
||||
let resolved = match resolve_data_backed_auth_context(
|
||||
let resolved = match resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -708,11 +733,12 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
}
|
||||
}
|
||||
AuthContextInflightRegistration::Bypass => {
|
||||
return resolve_data_backed_auth_context(
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -768,28 +794,39 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
return Ok((auth_context, None));
|
||||
}
|
||||
|
||||
let verified_api_key_hash = auth_context.verified_api_key_hash.clone();
|
||||
let snapshot = {
|
||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
if let Some(key_hash) = verified_api_key_hash.as_ref() {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_by_key_hash_strong(
|
||||
key_hash.as_str(),
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
} else {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
};
|
||||
let Some(snapshot) = snapshot else {
|
||||
let mut denied = auth_context;
|
||||
denied.access_allowed = false;
|
||||
denied.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
denied.balance_remaining = None;
|
||||
return Ok((denied, None));
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
if snapshot.user_id != auth_context.user_id || snapshot.api_key_id != auth_context.api_key_id {
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
let refreshed = build_data_backed_auth_context(
|
||||
let mut refreshed = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot.clone(),
|
||||
auth_endpoint_signature,
|
||||
@@ -798,9 +835,19 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
refreshed.verified_api_key_hash = verified_api_key_hash;
|
||||
Ok((refreshed, Some(snapshot)))
|
||||
}
|
||||
|
||||
fn deny_refreshed_auth_context(
|
||||
mut auth_context: GatewayControlAuthContext,
|
||||
) -> GatewayControlAuthContext {
|
||||
auth_context.access_allowed = false;
|
||||
auth_context.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
auth_context.balance_remaining = None;
|
||||
auth_context
|
||||
}
|
||||
|
||||
fn put_cached_auth_context(
|
||||
state: &AppState,
|
||||
cache_key: String,
|
||||
@@ -913,6 +960,23 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(signature) = auth_endpoint_signature
|
||||
.map(str::trim)
|
||||
@@ -923,7 +987,12 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
if !state.has_auth_api_key_reader() {
|
||||
return Ok(None);
|
||||
}
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let principal = derive_principal_candidate(&extracted);
|
||||
let now_unix_secs = current_unix_secs();
|
||||
|
||||
@@ -955,6 +1024,7 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -963,17 +1033,17 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
.await;
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
Ok(Some(
|
||||
build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await,
|
||||
))
|
||||
let mut auth_context = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
auth_context.verified_api_key_hash = Some(VerifiedApiKeyHash::new(key_hash));
|
||||
Ok(Some(auth_context))
|
||||
}
|
||||
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
||||
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
||||
@@ -1068,6 +1138,7 @@ async fn resolve_antigravity_bearer_bridge_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1127,6 +1198,7 @@ async fn resolve_trusted_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1158,9 +1230,7 @@ async fn build_data_backed_auth_context(
|
||||
let invalid_api_key = !snapshot.user_is_active
|
||||
|| snapshot.user_is_deleted
|
||||
|| !snapshot.api_key_is_active
|
||||
|| snapshot
|
||||
.api_key_expires_at_unix_secs
|
||||
.is_some_and(|expires_at| expires_at < current_unix_secs());
|
||||
|| api_key_is_expired(snapshot.api_key_expires_at_unix_secs, current_unix_secs());
|
||||
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
||||
let key_access_allowed = header_access_allowed
|
||||
.map(|value| value && snapshot.currently_usable)
|
||||
@@ -1225,9 +1295,14 @@ async fn build_data_backed_auth_context(
|
||||
local_rejection,
|
||||
allowed_models,
|
||||
ip_rules: snapshot.api_key_ip_rules,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn api_key_is_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
|
||||
expires_at_unix_secs.is_some_and(|expires_at| expires_at <= now_unix_secs)
|
||||
}
|
||||
|
||||
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
||||
items.iter().any(|item| api_format_matches(item, target))
|
||||
}
|
||||
@@ -1282,18 +1357,21 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
return true;
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return true;
|
||||
debug!(
|
||||
"deny requested provider {}: provider catalog is unavailable for allowlist resolution",
|
||||
requested_provider
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let providers = match state.list_provider_catalog_providers(true).await {
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider,
|
||||
err
|
||||
warn!(
|
||||
"deny requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1331,11 +1409,11 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
warn!(
|
||||
"deny requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1421,12 +1499,12 @@ fn get_cached_auth_context_with_age(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_data::repository::auth::{
|
||||
AuthApiKeyWriteRepository, InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||
AuthApiKeyWriteRepository, CreateUserApiKeyRecord, InMemoryAuthApiKeySnapshotRepository,
|
||||
StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::wallet::{
|
||||
@@ -1441,9 +1519,10 @@ mod tests {
|
||||
use futures_util::future::join_all;
|
||||
|
||||
use super::{
|
||||
get_cached_auth_context, resolve_control_decision_auth, resolve_data_backed_auth_context,
|
||||
resolve_execution_runtime_auth_context, ControlDecisionAuthResolution,
|
||||
GatewayLocalAuthRejection,
|
||||
api_key_is_expired, get_cached_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, resolve_control_decision_auth,
|
||||
resolve_data_backed_auth_context, resolve_execution_runtime_auth_context,
|
||||
ControlDecisionAuthResolution, GatewayLocalAuthRejection,
|
||||
};
|
||||
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
||||
use crate::control::GatewayControlDecision;
|
||||
@@ -1481,6 +1560,14 @@ mod tests {
|
||||
path.parse().expect("uri should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_expiry_is_inclusive_at_the_declared_second() {
|
||||
assert!(!api_key_is_expired(None, 100));
|
||||
assert!(!api_key_is_expired(Some(101), 100));
|
||||
assert!(api_key_is_expired(Some(100), 100));
|
||||
assert!(api_key_is_expired(Some(99), 100));
|
||||
}
|
||||
|
||||
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
id.to_string(),
|
||||
@@ -1507,40 +1594,24 @@ mod tests {
|
||||
.expect("endpoint should build")
|
||||
}
|
||||
|
||||
struct SqliteAuthConfigNodes {
|
||||
struct PostgresAuthConfigNodes {
|
||||
first: AppState,
|
||||
second: AppState,
|
||||
_database: TemporarySqliteDatabase,
|
||||
_database: aether_testkit::ManagedPostgresServer,
|
||||
}
|
||||
|
||||
struct TemporarySqliteDatabase(PathBuf);
|
||||
|
||||
impl Drop for TemporarySqliteDatabase {
|
||||
fn drop(&mut self) {
|
||||
for suffix in ["", "-shm", "-wal"] {
|
||||
let mut path = self.0.as_os_str().to_os_string();
|
||||
path.push(suffix);
|
||||
let _ = std::fs::remove_file(PathBuf::from(path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sqlite_auth_config_nodes(
|
||||
async fn postgres_auth_config_nodes(
|
||||
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
||||
) -> SqliteAuthConfigNodes {
|
||||
let database_path = std::env::temp_dir().join(format!(
|
||||
"aether-auth-config-{}.sqlite",
|
||||
uuid::Uuid::new_v4()
|
||||
));
|
||||
) -> PostgresAuthConfigNodes {
|
||||
let server = aether_testkit::ManagedPostgresServer::start()
|
||||
.await
|
||||
.expect("temporary PostgreSQL should start");
|
||||
let mut pool = SqlPoolConfig::default();
|
||||
pool.min_connections = 0;
|
||||
pool.max_connections = 4;
|
||||
let database = SqlDatabaseConfig::new(
|
||||
DatabaseDriver::Sqlite,
|
||||
format!("sqlite://{}", database_path.display()),
|
||||
pool,
|
||||
)
|
||||
.expect("sqlite config should build");
|
||||
let database =
|
||||
SqlDatabaseConfig::new(DatabaseDriver::Postgres, server.database_url(), pool)
|
||||
.expect("postgres config should build");
|
||||
let config = GatewayDataConfig::from_database_config(database);
|
||||
let first_data = GatewayDataState::from_config(config.clone())
|
||||
.expect("first data state should build")
|
||||
@@ -1549,27 +1620,27 @@ mod tests {
|
||||
assert!(first_data
|
||||
.run_database_migrations()
|
||||
.await
|
||||
.expect("sqlite migrations should run"));
|
||||
.expect("postgres migrations should run"));
|
||||
let second_data = GatewayDataState::from_config(config)
|
||||
.expect("second data state should build")
|
||||
.with_auth_api_key_reader(auth_repository)
|
||||
.without_wallet_reader_for_tests();
|
||||
|
||||
SqliteAuthConfigNodes {
|
||||
PostgresAuthConfigNodes {
|
||||
first: AppState::new()
|
||||
.expect("first app state should build")
|
||||
.with_data_state_for_tests(first_data),
|
||||
second: AppState::new()
|
||||
.expect("second app state should build")
|
||||
.with_data_state_for_tests(second_data),
|
||||
_database: TemporarySqliteDatabase(database_path),
|
||||
_database: server,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
||||
let nodes =
|
||||
sqlite_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||
postgres_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||
.await;
|
||||
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
||||
let old_value = serde_json::json!({"version": "old"});
|
||||
@@ -1769,6 +1840,97 @@ mod tests {
|
||||
assert_eq!(repository.touch_count("key-1"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn long_lived_refresh_rejects_same_ids_recreated_with_a_different_credential() {
|
||||
let old_api_key = "sk-old-websocket-credential";
|
||||
let new_api_key = "sk-new-websocket-credential";
|
||||
let old_key_hash = hash_api_key(old_api_key);
|
||||
let new_key_hash = hash_api_key(new_api_key);
|
||||
let mut old_snapshot = sample_snapshot("key-stable-id", "user-stable-id");
|
||||
old_snapshot.user_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
old_snapshot.api_key_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(old_key_hash.clone()),
|
||||
old_snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_repository_for_tests(repository.clone());
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
format!("Bearer {old_api_key}").parse().unwrap(),
|
||||
);
|
||||
|
||||
let original = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/responses"),
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("initial auth resolution should succeed")
|
||||
.expect("the old API key should authenticate");
|
||||
assert!(original.access_allowed);
|
||||
assert!(original.verified_api_key_hash.is_some());
|
||||
assert!(
|
||||
!format!("{original:?}").contains(&old_key_hash),
|
||||
"the credential verifier must stay redacted from Debug output"
|
||||
);
|
||||
|
||||
assert!(repository
|
||||
.delete_user_api_key("user-stable-id", "key-stable-id")
|
||||
.await
|
||||
.expect("old API key deletion should succeed"));
|
||||
repository
|
||||
.create_user_api_key(CreateUserApiKeyRecord {
|
||||
user_id: "user-stable-id".to_string(),
|
||||
api_key_id: "key-stable-id".to_string(),
|
||||
key_hash: new_key_hash,
|
||||
key_encrypted: None,
|
||||
name: Some("restored-with-new-secret".to_string()),
|
||||
allowed_providers: Some(vec!["openai".to_string()]),
|
||||
allowed_api_formats: Some(vec!["openai:responses".to_string()]),
|
||||
allowed_models: Some(vec!["gpt-4.1".to_string()]),
|
||||
ip_rules: None,
|
||||
rate_limit: 60,
|
||||
concurrent_limit: Some(5),
|
||||
force_capabilities: None,
|
||||
feature_settings: None,
|
||||
is_active: true,
|
||||
expires_at_unix_secs: Some(4_102_444_800),
|
||||
auto_delete_on_expiry: false,
|
||||
total_requests: 0,
|
||||
total_tokens: 0,
|
||||
total_cost_usd: 0.0,
|
||||
})
|
||||
.await
|
||||
.expect("same-ID API key recreation should resolve")
|
||||
.expect("same-ID API key recreation should persist");
|
||||
|
||||
let (refreshed, snapshot) = refresh_execution_runtime_auth_context_with_snapshot(
|
||||
&state,
|
||||
original,
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("long-lived auth refresh should resolve");
|
||||
|
||||
assert!(!refreshed.access_allowed);
|
||||
assert_eq!(
|
||||
refreshed.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::InvalidApiKey)
|
||||
);
|
||||
assert!(snapshot.is_none());
|
||||
assert_eq!(repository.key_hash_lookup_count(&old_key_hash), 1);
|
||||
assert_eq!(
|
||||
repository.snapshot_lookup_count("key-stable-id"),
|
||||
0,
|
||||
"a bound long-lived credential must not fall back to identity-only lookup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
||||
let api_key = "sk-test-concurrent-auth-miss";
|
||||
@@ -2396,6 +2558,44 @@ mod tests {
|
||||
assert_eq!(auth_context.local_rejection, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_backed_auth_context_denies_unresolved_provider_id_without_catalog_reader() {
|
||||
let api_key = "sk-test-provider-no-catalog";
|
||||
let mut snapshot = sample_snapshot("key-no-catalog", "user-no-catalog");
|
||||
snapshot.user_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.api_key_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.user_allowed_api_formats = None;
|
||||
snapshot.api_key_allowed_api_formats = None;
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(hash_api_key(api_key)),
|
||||
snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-api-key", api_key.parse().unwrap());
|
||||
|
||||
let auth_context = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/messages"),
|
||||
Some("claude:messages"),
|
||||
)
|
||||
.await
|
||||
.expect("resolution should succeed")
|
||||
.expect("auth context should exist");
|
||||
|
||||
assert!(!auth_context.access_allowed);
|
||||
assert_eq!(
|
||||
auth_context.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::ProviderNotAllowed {
|
||||
provider: "claude".to_string(),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
||||
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
||||
@@ -2410,7 +2610,7 @@ mod tests {
|
||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
None, snapshot,
|
||||
)]));
|
||||
let nodes = sqlite_auth_config_nodes(auth_repository.clone()).await;
|
||||
let nodes = postgres_auth_config_nodes(auth_repository.clone()).await;
|
||||
nodes
|
||||
.first
|
||||
.auth_context_cache
|
||||
|
||||
@@ -44,7 +44,7 @@ pub(super) struct GatewayTrustedAdminHeaders {
|
||||
pub(super) management_token_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Default, PartialEq, Eq)]
|
||||
pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) authorization_bearer: Option<String>,
|
||||
pub(super) x_api_key: Option<String>,
|
||||
@@ -54,7 +54,25 @@ pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) cookie_header: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
impl std::fmt::Debug for GatewayCredentialBundle {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let redacted = |value: &Option<String>| value.as_ref().map(|_| "[REDACTED]");
|
||||
formatter
|
||||
.debug_struct("GatewayCredentialBundle")
|
||||
.field(
|
||||
"authorization_bearer",
|
||||
&redacted(&self.authorization_bearer),
|
||||
)
|
||||
.field("x_api_key", &redacted(&self.x_api_key))
|
||||
.field("api_key", &redacted(&self.api_key))
|
||||
.field("x_goog_api_key", &redacted(&self.x_goog_api_key))
|
||||
.field("query_key", &redacted(&self.query_key))
|
||||
.field("cookie_header", &redacted(&self.cookie_header))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(super) enum GatewayPrimaryCredential {
|
||||
ProviderApiKey {
|
||||
raw: String,
|
||||
@@ -70,6 +88,21 @@ pub(super) enum GatewayPrimaryCredential {
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrimaryCredential {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (variant, carrier) = match self {
|
||||
Self::ProviderApiKey { carrier, .. } => ("ProviderApiKey", carrier),
|
||||
Self::BearerToken { carrier, .. } => ("BearerToken", carrier),
|
||||
Self::CookieHeader { carrier, .. } => ("CookieHeader", carrier),
|
||||
};
|
||||
formatter
|
||||
.debug_struct(variant)
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
||||
@@ -78,7 +111,7 @@ pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) primary: Option<GatewayPrimaryCredential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub(super) enum GatewayPrincipalCandidate {
|
||||
TrustedHeaders(GatewayTrustedAuthHeaders),
|
||||
ApiKeyHash {
|
||||
@@ -94,3 +127,58 @@ pub(super) enum GatewayPrincipalCandidate {
|
||||
carrier: GatewayCredentialCarrier,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrincipalCandidate {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::TrustedHeaders(headers) => formatter
|
||||
.debug_tuple("TrustedHeaders")
|
||||
.field(headers)
|
||||
.finish(),
|
||||
Self::ApiKeyHash { carrier, .. } => formatter
|
||||
.debug_struct("ApiKeyHash")
|
||||
.field("key_hash", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredBearerToken { carrier, .. } => formatter
|
||||
.debug_struct("DeferredBearerToken")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredCookieHeader { carrier, .. } => formatter
|
||||
.debug_struct("DeferredCookieHeader")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod debug_redaction_tests {
|
||||
use super::{GatewayCredentialBundle, GatewayCredentialCarrier, GatewayPrimaryCredential};
|
||||
|
||||
#[test]
|
||||
fn gateway_credential_debug_output_redacts_raw_authorization_values() {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: Some("bundle-bearer-canary".to_string()),
|
||||
api_key: Some("bundle-api-key-canary".to_string()),
|
||||
cookie_header: Some("bundle-cookie-canary".to_string()),
|
||||
..GatewayCredentialBundle::default()
|
||||
};
|
||||
let primary = GatewayPrimaryCredential::ProviderApiKey {
|
||||
raw: "primary-api-key-canary".to_string(),
|
||||
carrier: GatewayCredentialCarrier::ApiKey,
|
||||
};
|
||||
let debug = format!("{bundle:?} {primary:?}");
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
for secret in [
|
||||
"bundle-bearer-canary",
|
||||
"bundle-api-key-canary",
|
||||
"bundle-cookie-canary",
|
||||
"primary-api-key-canary",
|
||||
] {
|
||||
assert!(!debug.contains(secret), "debug output leaked {secret}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,9 +8,10 @@ mod public;
|
||||
mod route;
|
||||
|
||||
pub(crate) use auth::{
|
||||
execution_plan_balance_capacity_rejection, extract_requested_model,
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
request_model_local_rejection, resolve_execution_runtime_auth_context,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
extract_requested_model, refresh_execution_runtime_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, request_model_local_rejection,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayCredentialCarrier,
|
||||
GatewayLocalAuthRejection,
|
||||
@@ -18,14 +19,16 @@ pub(crate) use auth::{
|
||||
pub(crate) use execute::{allows_control_execute_emergency, maybe_execute_via_control};
|
||||
pub(crate) use management_token_permissions::{
|
||||
all_assignable_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions, legacy_full_management_token_permissions,
|
||||
management_token_permission_catalog_payload, management_token_permission_keys_from_value,
|
||||
management_token_permission_mode_and_summary,
|
||||
management_token_permissions_cover_all_assignable_permissions,
|
||||
management_token_permission_mode_and_summary, management_token_principal_has_permission,
|
||||
management_token_required_permission, normalize_assignable_management_token_permissions,
|
||||
read_only_management_token_permissions, validate_management_token_admin_route_permission,
|
||||
};
|
||||
pub(crate) use public::{resolve_public_request_context, GatewayPublicRequestContext};
|
||||
pub(crate) use public::{
|
||||
resolve_public_request_context, resolve_public_request_context_with_trusted_auth,
|
||||
resolve_public_request_context_without_trusted_auth, GatewayPublicRequestContext,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use route::classify_control_route;
|
||||
pub(crate) use route::{resolve_control_route, GatewayControlDecision};
|
||||
|
||||
@@ -2,7 +2,9 @@ use axum::http::Uri;
|
||||
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{resolve_control_route, GatewayControlDecision};
|
||||
use super::{
|
||||
resolve_control_route, route::resolve_control_route_with_trusted_auth, GatewayControlDecision,
|
||||
};
|
||||
|
||||
pub(crate) type GatewayPublicRequestContext =
|
||||
aether_gateway_control::PublicRequestContext<GatewayControlDecision>;
|
||||
@@ -23,3 +25,41 @@ pub(crate) async fn resolve_public_request_context(
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, true)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_without_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, false)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -302,6 +302,19 @@ pub(super) fn classify_admin_endpoints_family_route(
|
||||
"admin:endpoints_manage",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& normalized_path
|
||||
.strip_prefix("/api/admin/endpoints/")
|
||||
.and_then(|path| path.strip_suffix("/rules/reveal"))
|
||||
.is_some_and(|endpoint_id| !endpoint_id.is_empty() && !endpoint_id.contains('/'))
|
||||
{
|
||||
Some(classified(
|
||||
"admin_proxy",
|
||||
"endpoints_manage",
|
||||
"reveal_endpoint_rules",
|
||||
"admin:endpoints_manage",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& normalized_path.starts_with("/api/admin/endpoints/")
|
||||
&& !normalized_path.starts_with("/api/admin/endpoints/health/")
|
||||
|
||||
@@ -596,6 +596,20 @@ pub(super) fn classify_admin_observability_family_route(
|
||||
"admin:stats",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
"/api/admin/stats/leaderboard/user-groups"
|
||||
| "/api/admin/stats/leaderboard/user-groups/"
|
||||
)
|
||||
{
|
||||
Some(classified(
|
||||
"admin_proxy",
|
||||
"stats_manage",
|
||||
"leaderboard_user_groups",
|
||||
"admin:stats",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user