mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-09 18:59:50 +08:00
Compare commits
330
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bad13237d2 | ||
|
|
b4c0d02e01 | ||
|
|
180da72b69 | ||
|
|
db110fe9dd | ||
|
|
58ec544aac | ||
|
|
911c7f8875 | ||
|
|
310098a853 | ||
|
|
466c7918a1 | ||
|
|
e7de935e61 | ||
|
|
b1943710b3 | ||
|
|
082348a5d1 | ||
|
|
ac7e3abab3 | ||
|
|
7acbaab82b | ||
|
|
716c35bf56 | ||
|
|
2539439656 | ||
|
|
e379bd791c | ||
|
|
84f0d5ce70 | ||
|
|
d4ed774423 | ||
|
|
0948f29da5 | ||
|
|
8a767c4309 | ||
|
|
bd35e885a7 | ||
|
|
8ffd8188e4 | ||
|
|
976c3feca6 | ||
|
|
5d5281b022 | ||
|
|
bda0d4e44d | ||
|
|
cb7b9c9ecd | ||
|
|
12d58327a1 | ||
|
|
2075cd95de | ||
|
|
625456ff24 | ||
|
|
e1dadf5b06 | ||
|
|
8f1e2decf2 | ||
|
|
9f829450d8 | ||
|
|
65f43a7aef | ||
|
|
2a63bafd20 | ||
|
|
ea6b739fd6 | ||
|
|
14befeda2c | ||
|
|
d13f52bdf8 | ||
|
|
2257c3959f | ||
|
|
066ea87d72 | ||
|
|
fae51a0ec8 | ||
|
|
db5d2dfbb5 | ||
|
|
54fbcc25a1 | ||
|
|
d068b62966 | ||
|
|
017d08d45e | ||
|
|
7f45617f79 | ||
|
|
9cc4018a37 | ||
|
|
b49f5c0fd7 | ||
|
|
125cd40aa5 | ||
|
|
491364867e | ||
|
|
8093899b5a | ||
|
|
fa06e9695d | ||
|
|
5d5880d75e | ||
|
|
9abbf268a1 | ||
|
|
e8ee7b4ecf | ||
|
|
c1aa5d618d | ||
|
|
309f507ef4 | ||
|
|
00a315e3bf | ||
|
|
f811e29f13 | ||
|
|
fb7e3fc224 | ||
|
|
cabca22d65 | ||
|
|
11989e278e | ||
|
|
532af797bf | ||
|
|
cafa05c4cb | ||
|
|
49ec53cbd9 | ||
|
|
73f1d79637 | ||
|
|
1b8f78a992 | ||
|
|
14ef19d9e1 | ||
|
|
bc0e9f94e2 | ||
|
|
74072e5007 | ||
|
|
c96f8272f4 | ||
|
|
4c07d9fcfb | ||
|
|
45678d9419 | ||
|
|
75bc32cfe9 | ||
|
|
d4bc058c2f | ||
|
|
2516e51b4e | ||
|
|
3541ccfe29 | ||
|
|
bd83cff58f | ||
|
|
85c04335c6 | ||
|
|
c08497c963 | ||
|
|
d30268f80f | ||
|
|
5679375f71 | ||
|
|
3465d23db3 | ||
|
|
e3333f1aef | ||
|
|
31d5e2d172 | ||
|
|
bb9f2eed8e | ||
|
|
926f5cc928 | ||
|
|
27ae884759 | ||
|
|
834eb9c308 | ||
|
|
390b73d4b6 | ||
|
|
bcd121d447 | ||
|
|
3394a51278 | ||
|
|
75471ae4a4 | ||
|
|
57f53903f5 | ||
|
|
81788d3a64 | ||
|
|
30bb0c3130 | ||
|
|
595b8e4e05 | ||
|
|
7e033d0571 | ||
|
|
5745442ed7 | ||
|
|
1a4eba1005 | ||
|
|
2a9d8d3b25 | ||
|
|
e25e240d16 | ||
|
|
cd765f2c2f | ||
|
|
ec95989e02 | ||
|
|
7f5e1a64fe | ||
|
|
f86dd10467 | ||
|
|
69930a6059 | ||
|
|
07cb401fd4 | ||
|
|
593327c803 | ||
|
|
a9a7c64e5d | ||
|
|
70d1a4ab74 | ||
|
|
0b7c7f94ac | ||
|
|
67d0414483 | ||
|
|
e3c01fb554 | ||
|
|
f960bbd2c8 | ||
|
|
906baae88e | ||
|
|
ba7c9f8b27 | ||
|
|
37e3a36680 | ||
|
|
166de33355 | ||
|
|
a95f0d2488 | ||
|
|
b296d46e97 | ||
|
|
0486435f16 | ||
|
|
fb25dde4c9 | ||
|
|
4124749a7d | ||
|
|
5a55116b62 | ||
|
|
681ce56c4f | ||
|
|
bcb2308000 | ||
|
|
6c92db2ba5 | ||
|
|
fdf55525f5 | ||
|
|
4ff4129034 | ||
|
|
364692da55 | ||
|
|
5842c7232e | ||
|
|
72a4bf3408 | ||
|
|
fe1723d87c | ||
|
|
03b198d5ab | ||
|
|
53562fd9de | ||
|
|
e66dd00b84 | ||
|
|
03496c46c5 | ||
|
|
c0ded116a9 | ||
|
|
e5ab73bf35 | ||
|
|
a5456cdc38 | ||
|
|
e9899200f8 | ||
|
|
5a6692ade0 | ||
|
|
6e431e2ff6 | ||
|
|
88df2a2ed5 | ||
|
|
e7864e5611 | ||
|
|
6e6407160a | ||
|
|
01acff0774 | ||
|
|
cc5050155f | ||
|
|
04c4a97766 | ||
|
|
e83399db2f | ||
|
|
f753f14fd2 | ||
|
|
7daf355e65 | ||
|
|
e4f89de90e | ||
|
|
dfe88e34e7 | ||
|
|
c5adcf0312 | ||
|
|
ea24d61910 | ||
|
|
23e0af7b12 | ||
|
|
60b89cc840 | ||
|
|
b748b5bfd2 | ||
|
|
30e36cd09a | ||
|
|
28cd77eb5e | ||
|
|
531f53b443 | ||
|
|
95e4d0149c | ||
|
|
72aea7898b | ||
|
|
8aedf87aad | ||
|
|
e9b64c3e9d | ||
|
|
28f61ec45b | ||
|
|
6aeadcd1d7 | ||
|
|
3a8dadcd6b | ||
|
|
ecc16673eb | ||
|
|
d28dd89039 | ||
|
|
33ea4ebf15 | ||
|
|
8260a87215 | ||
|
|
361952ada9 | ||
|
|
6630856061 | ||
|
|
a893bd0557 | ||
|
|
f2839ae6a7 | ||
|
|
e58570d79d | ||
|
|
99f6499b2b | ||
|
|
17d01d7fe0 | ||
|
|
8b766930b0 | ||
|
|
c7e403b410 | ||
|
|
cf8ea19856 | ||
|
|
7113d04f8a | ||
|
|
099b810a2f | ||
|
|
7aa0c89244 | ||
|
|
7847ae98c6 | ||
|
|
a90d564931 | ||
|
|
a5c3699ae9 | ||
|
|
7b8048c6ae | ||
|
|
ec95f2ca1f | ||
|
|
aa7dbe67d3 | ||
|
|
a26680f460 | ||
|
|
522b979052 | ||
|
|
808946312a | ||
|
|
741107bf71 | ||
|
|
6962731220 | ||
|
|
062e111c03 | ||
|
|
470c59e197 | ||
|
|
2f929e74c7 | ||
|
|
fc0417ceb9 | ||
|
|
44174a31e0 | ||
|
|
b599fb7354 | ||
|
|
14f96c9fa0 | ||
|
|
6948852992 | ||
|
|
1b01b08c31 | ||
|
|
2281f2b754 | ||
|
|
b5ed802277 | ||
|
|
d1b5eb08ee | ||
|
|
dba5e6e9e9 | ||
|
|
c125e78c5f | ||
|
|
d1cb0ebecf | ||
|
|
9d7a0665c0 | ||
|
|
882bb43125 | ||
|
|
db6c522d60 | ||
|
|
e29442a06a | ||
|
|
e15ea0d5d3 | ||
|
|
2f374d6af2 | ||
|
|
4a356f4ea5 | ||
|
|
c7676d567d | ||
|
|
5ca4f87951 | ||
|
|
1fee8954cc | ||
|
|
f69b770f5e | ||
|
|
856accdced | ||
|
|
92749b4d6e | ||
|
|
f08c2e6729 | ||
|
|
e420bc6324 | ||
|
|
d723fb92d3 | ||
|
|
5b1de5f921 | ||
|
|
7ed48e7b58 | ||
|
|
af712ebdbf | ||
|
|
33d5cd5993 | ||
|
|
f5e1420ee6 | ||
|
|
b37b252b14 | ||
|
|
0097ea89ad | ||
|
|
10e63507f0 | ||
|
|
9ff4d73d5c | ||
|
|
0e3bd7eff4 | ||
|
|
1c89b5f9ab | ||
|
|
cdbbda40a6 | ||
|
|
29a9d608d9 | ||
|
|
a6dc43d5f6 | ||
|
|
c6718754d3 | ||
|
|
afdd033745 | ||
|
|
d5f54ffe8b | ||
|
|
f5ec76c5c8 | ||
|
|
784a1e0611 | ||
|
|
507cb33089 | ||
|
|
b08fa3bdb6 | ||
|
|
018af84d7d | ||
|
|
27b0381a9a | ||
|
|
57cdef4b8d | ||
|
|
36e9d21e3f | ||
|
|
b72b6ab137 | ||
|
|
30b2c8548a | ||
|
|
7c5cce4b3c | ||
|
|
9362c34fcd | ||
|
|
344b3031e9 | ||
|
|
e89c3aa674 | ||
|
|
ddbbf835af | ||
|
|
cb58a63ee3 | ||
|
|
6b1074cfcd | ||
|
|
d6894b5532 | ||
|
|
635c6765d9 | ||
|
|
86f7cc0d58 | ||
|
|
206995645b | ||
|
|
9282cce1d6 | ||
|
|
c005700a7e | ||
|
|
14744abd57 | ||
|
|
66d6c17d2d | ||
|
|
c142d39951 | ||
|
|
1eb2d10dec | ||
|
|
dabaeb8dfa | ||
|
|
2d17d4b73f | ||
|
|
18d78dd6c9 | ||
|
|
499942e3e7 | ||
|
|
ba11a72214 | ||
|
|
12571764bc | ||
|
|
1e13fa032c | ||
|
|
47b21a25d3 | ||
|
|
45a3ba8829 | ||
|
|
03f2914044 | ||
|
|
c8d1ae3e7e | ||
|
|
c5ae9c2c77 | ||
|
|
4e47c00154 | ||
|
|
313a637982 | ||
|
|
fe8ff268df | ||
|
|
bac6d6866a | ||
|
|
579f2c7cc1 | ||
|
|
ddcbeb3ae9 | ||
|
|
e25fc984af | ||
|
|
4cf47b1dee | ||
|
|
95cbd43097 | ||
|
|
09005939bf | ||
|
|
7b612b8b5a | ||
|
|
670d5e8d33 | ||
|
|
1de2e70d41 | ||
|
|
89b57464d2 | ||
|
|
09ef3adf70 | ||
|
|
3dfc15963c | ||
|
|
f6884eb8c4 | ||
|
|
f8b4382a54 | ||
|
|
d78b5a81fb | ||
|
|
89fe9e9f0a | ||
|
|
4291a91dc0 | ||
|
|
4c6bafe255 | ||
|
|
979dbc4b33 | ||
|
|
9309ad844f | ||
|
|
d672ba2068 | ||
|
|
587486ab0c | ||
|
|
40a5e1470d | ||
|
|
058660ec2e | ||
|
|
668bf5e40f | ||
|
|
77f93c638d | ||
|
|
d0c0996b9f | ||
|
|
2cb4d554aa | ||
|
|
76fb8905c9 | ||
|
|
f822df6cce | ||
|
|
45c840b8d3 | ||
|
|
e8d9877b79 | ||
|
|
cae9aa4134 | ||
|
|
e3644c6142 | ||
|
|
cc6f5e89b6 | ||
|
|
2ed2cc66ef | ||
|
|
b1bf7837cf | ||
|
|
77229943d1 | ||
|
|
a0369cf49a | ||
|
|
dbbe7b22ab | ||
|
|
d5f34b2ee2 | ||
|
|
5059093d29 |
@@ -0,0 +1,47 @@
|
||||
# Agent Note: 成本分析“提供商用量”展示提供商名称
|
||||
|
||||
Status: implemented
|
||||
|
||||
## Problem
|
||||
|
||||
成本分析页面(`frontend/src/views/admin/CostAnalysis.vue` → `data-provider-usage`)的“提供商”列直接渲染接口返回的 `label`。而后端 `UsageAnalyticsView::Breakdown` + `group_by=provider` 的分组键是 `provider_id`,同一段 SQL 又把 `label` 写成 `group_id::text`,于是页面显示 `provider-1` 这类内部 ID,管理员无法判断是哪个提供商;同一份数据的 CSV 导出 `label` 列也只有 ID。
|
||||
|
||||
不修的话,这个缺陷不会自愈:`id` 必须继续是 `provider_id`(“查看”链接按它跳转用量明细、已登记支出按它匹配),所以只能修 `label` 的取值,而不是换分组键。故障面覆盖两套实现:PostgreSQL 适配器与内存仓储(无数据库部署与单元测试走后者)。
|
||||
|
||||
## Decision
|
||||
|
||||
`Breakdown` 提供商分组下,行的 `id` 保持 `provider_id` 不变,`label` 解析为提供商名称,解析顺序:
|
||||
|
||||
1. 提供商目录 `providers.name`(当前名称,随改名刷新);
|
||||
2. 使用记录里的 `provider_name` 快照(历史行、目录已删的提供商);
|
||||
3. 原始 `provider_id`(最后的兜底,避免出现空列)。
|
||||
|
||||
配套边界,两套实现一致:
|
||||
|
||||
- 空白字符串与历史占位值 `unknown` / `unknow` / `pending` 不当作名称展示;
|
||||
- `provider_id` 为空表示这条记录无法归属,`label` 保持 `null`,由前端沿用既有的“未归属提供商”文案;
|
||||
- 只有明细分组(`Breakdown`)这么做;时间序列 / Performance / DashboardCharts 仍用 `provider_id` 作标签,`provider_rows` 与 `provider_timeline_rows` 的既有口径不变;
|
||||
- 内存仓储没有提供商目录,只走第 2、3 步,分组键与标签规则与 PostgreSQL 侧保持一致。
|
||||
|
||||
PostgreSQL 侧的落地方式:`grouped` CTE 用 `max(provider_name) AS provider_name` 带出名称快照,最外层 `FROM page LEFT JOIN public.providers AS provider_catalog ON provider_catalog.id = page.group_id`,标签表达式固定在 `ANALYTICS_PROVIDER_LABEL_SQL`;`to_jsonb(page)` 需要额外减去 `provider_name`,否则它会被塞进 `metrics`。
|
||||
|
||||
前端不改:`label` 为空时已经回退到“未归属提供商”,`id` 与 `label` 的职责在页面里本来就是分开的。
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **前端用已加载的财务账户列表(`providerFinanceApi.accounts`)把 ID 映射成名称** — 改动最小且不动后端,但只对已登记财务账户的提供商有效,未登记账户的提供商依旧显示 ID;缺陷被藏在展示层,导出的 CSV 仍然只有 ID。否。
|
||||
- **只取使用记录里的 `provider_name` 快照(与 `provider_rows`、`provider_timeline_rows` 的 `max(provider_name)` 保持一致)** — 无需 JOIN,SQL 更短;但提供商在目录里改名后,历史报表仍显示旧名称,与用量审计聚合页的解析顺序不一致。目录优先、快照兜底。
|
||||
- **把分组键换成 `provider_name`(照搬用量审计聚合的 `legacy_name` 回退)** — 能让无法归属的历史行按名称分桶,但会改变 `total`、分页口径和“查看”链接语义,并让按 `provider_id` 匹配的已登记支出列错位。否。
|
||||
|
||||
## Consequences
|
||||
|
||||
- **收益**:页面的“提供商”列与 CSV 的 `label` 列显示可读名称,且与用量审计聚合(`USAGE_RESOLVED_PROVIDER_DISPLAY_NAME_SQL`)保持同一套解析顺序,跨页面观感一致。
|
||||
- **代价与已知上限**:明细查询多了一次 `providers` 的 LEFT JOIN(发生在 `LIMIT` 之后的 page 上,最多 `limit` 行,按主键关联);查询结果依赖 `providers` 表可见;提供商被删除后回退到快照名,历史报表不会因目录改名而重写既有数据。
|
||||
- **重访信号**:如果将来要求“同名历史提供商合并成一行”(即 `legacy_name` 归并),那是分组键变更,需要另开一篇笔记并同步改动 `total` / 分页 / 前端链接。
|
||||
|
||||
## Verification
|
||||
|
||||
- 内存实现:`cargo test -p aether-data --lib` → `repository::usage::memory::tests::overview_provider_breakdown_labels_rows_with_provider_name`(覆盖名称展示、占位值回退、无法归属三种分支)。
|
||||
- SQL 片段解析顺序:`cargo test -p aether-data-postgres --lib` → `usage::tests::provider_breakdown_labels_resolve_catalog_name_before_recorded_name_and_id`。
|
||||
- 展示层:`frontend/src/features/overview/__tests__/costs.spec.ts` 已断言“提供商用量”区块渲染 `Provider One`。
|
||||
- 取数端到端(真实 SQL 执行)需要 `AETHER_TEST_DATABASE_URL` 的 live 测试环境;本环境无可用 PostgreSQL,未执行 `crates/aether-data/adapters/postgres/src/usage/analytics_tests.rs` 中的 live 用例。
|
||||
@@ -0,0 +1,6 @@
|
||||
# 不固定 test-threads:nextest 默认按 num-cpus 并发,固定值会在更大规格的
|
||||
# runner 或本地开发机上主动压低并发、反而变慢,且无法表达 min(4, num-cpus)。
|
||||
# 这里只保留卡死保护,避免单个挂起用例拖满整个 job。
|
||||
[profile.default]
|
||||
# 60 秒后标记慢测试,连续两轮仍未结束则终止;超时结果保持失败,不隐藏回归。
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
+99
-11
@@ -26,21 +26,25 @@ RUST_LOG=aether_gateway=info
|
||||
# 示例: http://localhost:5173,https://app.example.com
|
||||
# CORS_ORIGINS=http://localhost:5173
|
||||
# CORS_ALLOW_CREDENTIALS=true
|
||||
# 如果前后端跨站并依赖登录刷新 Cookie,还要配合:
|
||||
# 登录刷新 Cookie 对同源浏览器请求和可信反代自动适配 HTTP/HTTPS。
|
||||
# HTTP 自动使用兼容的 SameSite=Lax(显式 Strict 保留);HTTPS 保留原有 SameSite 配置。
|
||||
# 无法确认访问协议时保留安全默认值;HTTPS 反代请正确传递 X-Forwarded-Proto。
|
||||
# AUTH_REFRESH_COOKIE_SECURE 可显式覆盖自动判断,公网部署仍建议使用 HTTPS。
|
||||
# 如果前后端跨站并依赖登录刷新 Cookie,必须使用 HTTPS,并配合:
|
||||
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
||||
# AUTH_REFRESH_COOKIE_SECURE=true
|
||||
|
||||
# 数据库配置
|
||||
# 数据库配置(仅支持 PostgreSQL)
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_NAME=aether
|
||||
DB_PASSWORD=aether
|
||||
DB_PASSWORD=
|
||||
|
||||
# Redis 配置
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=aether
|
||||
REDIS_PASSWORD=
|
||||
|
||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||
@@ -50,8 +54,12 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||
|
||||
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||
|
||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||
# 手动部署时取消注释并设置;install.sh 首次生成配置时会提示输入。
|
||||
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||
ADMIN_EMAIL=[email protected]
|
||||
ADMIN_USERNAME=admin123456
|
||||
# ADMIN_PASSWORD=
|
||||
@@ -69,21 +77,101 @@ ADMIN_USERNAME=admin123456
|
||||
# AETHER_VSCODEX_PUBLIC_WS_URL=wss://aether.example.com/api/vscodex/ws
|
||||
# AETHER_VSCODEX_ALLOWED_ORIGINS=https://aether.example.com
|
||||
|
||||
# docker compose 下 app 启动前自动执行 pending migration/backfill(默认 true)
|
||||
# AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
|
||||
# 启动时的数据库准备策略:auto(默认)或 verify-only
|
||||
# AETHER_GATEWAY_DATABASE_MODE=auto
|
||||
|
||||
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||
# 普通 PostgreSQL 连接默认语句超时 30 秒、锁等待超时 3 秒;0 显式关闭。
|
||||
# 这是单条 SQL 的期限,不是整个事务总期限;迁移和历史 backfill 使用独立连接放宽。
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS=30000
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS=3000
|
||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||
# 所有监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和升级后的 WebSocket。
|
||||
# 未设置或 0 时按请求上限 + WebSocket 上限推导,最大 65536;显式配置也受 FD 余量限制。
|
||||
# 已知 FD soft limit 时最多 max(1, (FD - 256) / 2),不是整个进程的 FD/内存保证。
|
||||
# 满额的新连接在 HTTP 解析前关闭,不排队创建任务,也不会返回 HTTP 429/503。
|
||||
# AETHER_GATEWAY_MAX_HTTP_CONNECTIONS=4096
|
||||
# 停机先等待 HTTP 请求,再排空本地用量写入;以下期限单位为毫秒。
|
||||
# 进程管理器的强杀期限应覆盖两阶段之和,再预留至少 10 秒收尾。
|
||||
# AETHER_GATEWAY_HTTP_SHUTDOWN_TIMEOUT_MS=30000
|
||||
# AETHER_GATEWAY_USAGE_SHUTDOWN_TIMEOUT_MS=30000
|
||||
# 每客户端、每 origin 的上游空闲连接缓存;不限制活动请求或流持续时间。
|
||||
# AETHER_GATEWAY_UPSTREAM_POOL_MAX_IDLE_PER_HOST=32
|
||||
# AETHER_GATEWAY_UPSTREAM_POOL_IDLE_TIMEOUT_MS=15000
|
||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||
# 请求体按实际缓冲增长申请额度,解压同时计入输入和输出;额度不足返回 503。
|
||||
# 请求体完整读取总超时默认 120000 毫秒;非零值限制在 1000-600000,显式 0 关闭。
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
||||
# 上游流首包后空闲超时默认 300000 毫秒;执行配置 read_ms 优先,显式 0 关闭。
|
||||
# AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS=300000
|
||||
# 流式响应诊断捕获共享预算默认 128 MiB,包含 provider/client 分配;不足时截断审计副本。
|
||||
# 显式 0 关闭此类捕获;不限制协议解析、终态编码和 usage 队列的总内存。
|
||||
# AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||
# usage 诊断正文共享预算,按 JSON 堆内存估算,默认 128 MiB。
|
||||
# 覆盖终态队列 seed、Redis 解码事件、数据库写入 DTO 及正文副本;额度随正文释放。
|
||||
# 不足或显式 0 时先保留计费事实再舍弃正文;已有清空/禁用状态保留,其余标为截断。
|
||||
# 不包含原始 Redis 批次、解码临时分配、序列化和压缩结果、协议观察缓冲或进程总内存。
|
||||
# AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES=134217728
|
||||
# 新 usage 队列消息的完整 JSON payload 上限,默认 1 MiB;显式 0 非法。
|
||||
# 超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义则拒绝入队,终态尝试受限落库,失败即明确失败。
|
||||
# 不限制存量 Redis 消息、整个读取批次、DLQ 或进程总内存。
|
||||
# usage_runtime_queue_payload_* 降级/拒绝计数包含入队和重试预校验的编码尝试,不代表唯一事件数。
|
||||
# AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES=1048576
|
||||
# usage worker 读取/重领共用的逻辑 payload 预留:全进程默认 128 MiB,单批目标 8 MiB。
|
||||
# 按当前消息上限推导 COUNT,默认最多 8 条;预留覆盖整批处理和确认,额度不足等待。
|
||||
# 当前消息上限不能超过总预留额度;单批目标不足一条时仍读一条。0 或非法值回退默认。
|
||||
# 历史/其他生产者的大消息继续处理并计数,不是 RESP、实际堆内存或 DLQ 的硬上限。
|
||||
# AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES=134217728
|
||||
# AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES=8388608
|
||||
# DLQ 原文和最坏 JSON 编码独立预留默认 64 MiB,后台编码及写入最多同时 4 个任务。
|
||||
# 入场前一次预留,额度占满或单条超预算立即失败并保留 pending 原消息,后续重领。
|
||||
# 编码失败不阻塞同批其余正常消息;批次仍报告失败,只有成功项会被确认。
|
||||
# JSON 按字符串最多 6 倍转义保守估算;不截原账务字段,不包含 Redis 命令/连接副本或 RSS。
|
||||
# 0/非法值回退默认;bytes 最大约 4 GiB,jobs 最大 128。超大存量可能需调高额度后恢复。
|
||||
# AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES=67108864
|
||||
# AETHER_USAGE_DLQ_ENCODING_MAX_JOBS=4
|
||||
# 内置 Redis 死信转移要求 Redis 7+ 及 EVAL/TYPE/XPENDING/XADD/XACK/XDEL 权限。
|
||||
# stream 与 DLQ 不能同名;Cluster 还要求两键同 slot,现有默认键未自动迁移。
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式 0 仍受 256 MiB 硬上限保护。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
||||
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
|
||||
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
|
||||
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
|
||||
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||
# 跨网关 relay 解析到受控私有地址时才显式开启;默认关闭以防止被篡改的 attachment
|
||||
# 记录诱导网关向内网转发 relay 凭据。该开关不放宽普通 provider 的目标地址策略。
|
||||
# AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS=false
|
||||
# 更推荐按 relay 主机名精确放行私网部署(逗号分隔,大小写不敏感);不支持通配符/后缀。
|
||||
# AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST=gateway-a.internal,gateway-b.internal
|
||||
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||
# AETHER_BARK_ALLOW_HTTP=false
|
||||
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||
|
||||
# 普通 Provider 反代(包括 Provider OAuth)不按 DNS 地址过滤上游,兼容任意
|
||||
# Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。
|
||||
# URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。
|
||||
|
||||
# 可选 Provider OAuth 客户端。Gemini CLI 和 Antigravity 默认使用内置 native-app
|
||||
# 客户端凭据;自定义 client ID 时必须同时配置对应的 client secret。
|
||||
# 显式配置的 client secret 优先于默认值。
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||
|
||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||
|
||||
@@ -6,7 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: build-tunnel-${{ github.ref }}
|
||||
@@ -17,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Ensure tunnel tag matches Cargo version
|
||||
shell: bash
|
||||
@@ -78,10 +79,10 @@ jobs:
|
||||
use_cross: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
@@ -89,14 +90,14 @@ jobs:
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
workspaces: apps/aether-tunnel -> target
|
||||
key: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
working-directory: apps/aether-tunnel
|
||||
@@ -122,9 +123,10 @@ jobs:
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-tunnel-${{ matrix.name }}
|
||||
path: |
|
||||
@@ -137,9 +139,14 @@ jobs:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: artifacts
|
||||
@@ -148,6 +155,20 @@ jobs:
|
||||
working-directory: artifacts
|
||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||
|
||||
- name: Attest tunnel release provenance
|
||||
id: attest-release
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
artifacts/aether-tunnel-*.tar.gz
|
||||
artifacts/aether-tunnel-*.zip
|
||||
artifacts/SHA256SUMS.txt
|
||||
|
||||
- name: Bundle tunnel release provenance
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Delete stale draft releases for tag
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -170,12 +191,13 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
name: "${{ github.ref_name }}"
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
artifacts/aether-tunnel-*
|
||||
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
artifacts/SHA256SUMS.txt
|
||||
fail_on_unmatched_files: true
|
||||
|
||||
@@ -183,8 +205,10 @@ jobs:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: main
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
@@ -46,10 +44,10 @@ jobs:
|
||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
@@ -77,10 +75,10 @@ jobs:
|
||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: frontend/dist
|
||||
|
||||
@@ -90,7 +88,10 @@ jobs:
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
id-token: write
|
||||
pages: write
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
@@ -25,7 +25,6 @@ env:
|
||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: '1'
|
||||
GHCR_IMAGE: ghcr.io/fawney19/aether
|
||||
|
||||
jobs:
|
||||
source:
|
||||
@@ -36,6 +35,7 @@ jobs:
|
||||
sha: ${{ steps.snapshot.outputs.sha }}
|
||||
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
||||
date: ${{ steps.snapshot.outputs.date }}
|
||||
ghcr_image: ${{ steps.snapshot.outputs.ghcr_image }}
|
||||
steps:
|
||||
- name: Require main branch
|
||||
id: snapshot
|
||||
@@ -49,9 +49,13 @@ jobs:
|
||||
fi
|
||||
|
||||
sha="${GITHUB_SHA}"
|
||||
# Docker 镜像仓库名必须全小写;GitHub owner 可能保留大写,先统一规范化。
|
||||
repository_owner="${GITHUB_REPOSITORY%%/*}"
|
||||
repository_owner="${repository_owner,,}"
|
||||
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
||||
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
||||
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
||||
echo "ghcr_image=ghcr.io/${repository_owner}/aether" >> "${GITHUB_OUTPUT}"
|
||||
echo "Building main at ${sha}."
|
||||
|
||||
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
||||
@@ -59,6 +63,8 @@ jobs:
|
||||
name: Rust CI
|
||||
needs: source
|
||||
uses: ./.github/workflows/rust-ci.yml
|
||||
with:
|
||||
full_scope: true
|
||||
|
||||
rust_extended:
|
||||
name: Rust extended checks
|
||||
@@ -66,12 +72,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
@@ -79,13 +85,13 @@ jobs:
|
||||
run: rustc -Vv
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check all workspace targets
|
||||
env:
|
||||
@@ -112,12 +118,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -157,7 +163,7 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -171,12 +177,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -247,38 +253,26 @@ jobs:
|
||||
arch: arm64
|
||||
os: ubuntu-latest
|
||||
use_cross: true
|
||||
- name: macos-amd64
|
||||
target: x86_64-apple-darwin
|
||||
platform: macos
|
||||
arch: amd64
|
||||
os: macos-15-intel
|
||||
use_cross: false
|
||||
- name: macos-arm64
|
||||
target: aarch64-apple-darwin
|
||||
platform: macos
|
||||
arch: arm64
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build release binary
|
||||
env:
|
||||
@@ -295,7 +289,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -308,17 +302,19 @@ jobs:
|
||||
needs: [source, checks, build]
|
||||
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download Linux binaries and frontend
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -335,20 +331,20 @@ jobs:
|
||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push nightly image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -372,12 +368,12 @@ jobs:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download nightly artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -392,31 +388,29 @@ jobs:
|
||||
VERSION="nightly"
|
||||
|
||||
mkdir -p package release-assets
|
||||
for platform in linux macos; do
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p "${root}/bin" "${root}/frontend"
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-linux-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p "${root}/bin" "${root}/frontend"
|
||||
|
||||
install -m 0755 \
|
||||
"artifacts/nightly-gateway-${platform}-${arch}/aether-gateway" \
|
||||
"${root}/bin/aether-gateway"
|
||||
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
install -m 0755 \
|
||||
"artifacts/nightly-gateway-linux-${arch}/aether-gateway" \
|
||||
"${root}/bin/aether-gateway"
|
||||
cp -R artifacts/nightly-frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
|
||||
sed \
|
||||
@@ -426,15 +420,15 @@ jobs:
|
||||
chmod 0755 release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 4
|
||||
test "$(wc -l < release-assets/SHA256SUMS)" -eq 4
|
||||
test "$(find release-assets -maxdepth 1 -name '*.tar.gz' | wc -l)" -eq 2
|
||||
test "$(wc -l < release-assets/SHA256SUMS)" -eq 2
|
||||
(cd release-assets && sha256sum -c SHA256SUMS)
|
||||
for archive in release-assets/*.tar.gz; do
|
||||
tar -tzf "${archive}" >/dev/null
|
||||
done
|
||||
|
||||
- name: Upload nightly package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets/*
|
||||
@@ -452,7 +446,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download nightly package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets
|
||||
@@ -466,6 +460,7 @@ jobs:
|
||||
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
||||
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
||||
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -510,6 +505,15 @@ jobs:
|
||||
--repo "${REPOSITORY}" \
|
||||
--clobber
|
||||
|
||||
published_assets="$(gh release view "${RELEASE_TAG}" --repo "${REPOSITORY}" --json assets --jq '.assets[].name')"
|
||||
while IFS= read -r asset_name; do
|
||||
if [[ "${asset_name}" == aether-nightly-*.tar.gz && ! -f "release-assets/${asset_name}" ]]; then
|
||||
gh release delete-asset "${RELEASE_TAG}" "${asset_name}" \
|
||||
--repo "${REPOSITORY}" \
|
||||
--yes
|
||||
fi
|
||||
done <<<"${published_assets}"
|
||||
|
||||
# target_commitish does not move an existing git tag. Move the ref
|
||||
# only after the complete asset set is available.
|
||||
if gh api "repos/${REPOSITORY}/git/ref/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
|
||||
@@ -534,8 +538,6 @@ jobs:
|
||||
expected_assets=(
|
||||
aether-nightly-linux-amd64.tar.gz
|
||||
aether-nightly-linux-arm64.tar.gz
|
||||
aether-nightly-macos-amd64.tar.gz
|
||||
aether-nightly-macos-arm64.tar.gz
|
||||
SHA256SUMS
|
||||
install.sh
|
||||
)
|
||||
|
||||
@@ -6,8 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-aether-${{ github.ref }}
|
||||
@@ -70,10 +70,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -106,10 +106,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -159,7 +159,7 @@ jobs:
|
||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||
|
||||
- name: Upload VSIX artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||
@@ -186,35 +186,23 @@ jobs:
|
||||
arch: arm64
|
||||
os: ubuntu-latest
|
||||
use_cross: true
|
||||
- name: macos-amd64
|
||||
target: x86_64-apple-darwin
|
||||
platform: macos
|
||||
arch: amd64
|
||||
os: macos-15-intel
|
||||
use_cross: false
|
||||
- name: macos-arm64
|
||||
target: aarch64-apple-darwin
|
||||
platform: macos
|
||||
arch: arm64
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
@@ -230,7 +218,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -242,11 +230,17 @@ jobs:
|
||||
needs: [preflight, frontend, build]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -259,27 +253,27 @@ jobs:
|
||||
cp -r artifacts/frontend-dist dist/frontend
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
@@ -295,7 +289,8 @@ jobs:
|
||||
latest=false
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
id: push
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -304,15 +299,36 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Attest GHCR image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
- name: Attest Docker Hub image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
package:
|
||||
name: Release tarballs
|
||||
needs: [preflight, frontend, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -328,31 +344,29 @@ jobs:
|
||||
fi
|
||||
|
||||
mkdir -p package release-assets
|
||||
for platform in linux macos; do
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-${platform}-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p \
|
||||
"${root}/bin" \
|
||||
"${root}/frontend"
|
||||
for arch in amd64 arm64; do
|
||||
bundle="aether-${VERSION}-linux-${arch}"
|
||||
root="package/${bundle}"
|
||||
mkdir -p \
|
||||
"${root}/bin" \
|
||||
"${root}/frontend"
|
||||
|
||||
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
install -m 0755 "artifacts/aether-gateway-linux-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||
sed \
|
||||
-e "s/^SOURCE_REF=\"\${AETHER_SOURCE_REF:-main}\"/SOURCE_REF=\"\${AETHER_SOURCE_REF:-${SOURCE_REF}}\"/" \
|
||||
-e "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" \
|
||||
install.sh > "${root}/install.sh"
|
||||
chmod 0755 "${root}/install.sh"
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
install -m 0644 LICENSE "${root}/LICENSE"
|
||||
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
tar -C package -czf "release-assets/${bundle}.tar.gz" "${bundle}"
|
||||
done
|
||||
|
||||
sed \
|
||||
@@ -362,8 +376,24 @@ jobs:
|
||||
chmod +x release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
- name: Attest release package provenance
|
||||
id: attest-release
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/install.sh
|
||||
release-assets/SHA256SUMS
|
||||
|
||||
- name: Bundle release package provenance
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Upload release package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets/*
|
||||
@@ -375,15 +405,18 @@ jobs:
|
||||
needs: [preflight, docker, package, vscodex]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download release package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
|
||||
- name: Download VSIX artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: release-assets
|
||||
@@ -410,13 +443,14 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Publish GitHub Release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
generate_release_notes: true
|
||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||
files: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
release-assets/SHA256SUMS
|
||||
release-assets/install.sh
|
||||
release-assets/*.vsix
|
||||
|
||||
+315
-221
@@ -2,6 +2,12 @@ name: Rust CI
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
full_scope:
|
||||
description: "Run all Rust and shell scopes, used by Nightly"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -9,18 +15,58 @@ on:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
|
||||
concurrency:
|
||||
group: rust-ci-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
@@ -36,14 +82,106 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
fmt:
|
||||
name: Format
|
||||
changes:
|
||||
name: Detect Rust CI scope
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
rust: ${{ steps.scope.outputs.rust }}
|
||||
shell: ${{ steps.scope.outputs.shell }}
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Classify changed paths
|
||||
id: scope
|
||||
shell: bash
|
||||
env:
|
||||
RUST_CI_FULL_SCOPE: ${{ inputs.full_scope || false }}
|
||||
run: |
|
||||
# 任何命令失败都必须让本 job 失败,否则 git fetch/diff 出错后仍会写出
|
||||
# rust=false/shell=false,下游会误判为“无需测试”而假绿放行。
|
||||
set -euo pipefail
|
||||
|
||||
# Nightly 通过 workflow_call 显式传入 full_scope;普通 push/PR 只按源码和构建
|
||||
# 指纹触发 Rust jobs,安装脚本、Compose、README 等由 shell scope 覆盖。
|
||||
if [ "$RUST_CI_FULL_SCOPE" = "true" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] \
|
||||
&& [ -n "${GITHUB_BASE_REF:-}" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
git fetch --no-tags origin "$GITHUB_BASE_REF" --depth=1
|
||||
changed_paths=$(git diff --name-only "origin/$GITHUB_BASE_REF...$GITHUB_SHA")
|
||||
elif [ "$GITHUB_EVENT_NAME" = "push" ] \
|
||||
&& [ -n "${GITHUB_EVENT_BEFORE:-}" ] \
|
||||
&& [ "$GITHUB_EVENT_BEFORE" != "0000000000000000000000000000000000000000" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
changed_paths=$(git diff --name-only "$GITHUB_EVENT_BEFORE" "$GITHUB_SHA")
|
||||
else
|
||||
changed_paths=$(git ls-files)
|
||||
fi
|
||||
|
||||
# 防御性兜底:diff 结果为空(异常事件或比较失败)时按全量运行,
|
||||
# 宁可多跑也不能漏测。
|
||||
if [ -z "$changed_paths" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rust=false
|
||||
shell=false
|
||||
while IFS= read -r path; do
|
||||
case "$path" in
|
||||
Cargo.toml|Cargo.lock|rust-toolchain.toml|.cargo/*|*.rs|*/Cargo.toml|*/build.rs|*.sql|.github/workflows/*.yml|.github/workflows/*.yaml)
|
||||
rust=true
|
||||
;;
|
||||
*.sh|*.py|README.md|*/README.md|.env.example|Dockerfile*|docker-compose*.yml|docker-compose*.yaml)
|
||||
shell=true
|
||||
;;
|
||||
esac
|
||||
done <<< "$changed_paths"
|
||||
|
||||
echo "rust=$rust" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=$shell" >> "$GITHUB_OUTPUT"
|
||||
|
||||
shell_security:
|
||||
name: Shell security fixtures
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.shell == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Run installer and supply-chain fixtures
|
||||
shell: bash
|
||||
run: |
|
||||
python3 tests/compose_database_config_test.py
|
||||
bash tests/deploy_state_safety_test.sh
|
||||
bash tests/install_archive_safety_test.sh
|
||||
bash tests/install_container_runtime_security_test.sh
|
||||
bash tests/install_current_release_link_test.sh
|
||||
bash tests/install_local_bundle_safety_test.sh
|
||||
bash tests/install_privileged_write_safety_test.sh
|
||||
bash tests/install_source_trust_test.sh
|
||||
bash tests/release_supply_chain_test.sh
|
||||
bash tests/update_compose_safety_test.sh
|
||||
bash tests/tunnel_installer_config_security_test.sh
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: rustfmt
|
||||
@@ -53,24 +191,28 @@ jobs:
|
||||
|
||||
clippy_gateway:
|
||||
name: Clippy (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# Gateway lint 与 Gateway 测试都可能触发 mold/大型链接依赖,单独隔离缓存
|
||||
# 指纹,避免不同 job 的构建产物互相驱逐或复用错误的链接参数。
|
||||
shared-key: rust-ci-gateway-clippy-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -87,24 +229,26 @@ jobs:
|
||||
|
||||
clippy_data:
|
||||
name: Clippy (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -121,24 +265,26 @@ jobs:
|
||||
|
||||
clippy_rest:
|
||||
name: Clippy (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -157,6 +303,7 @@ jobs:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- clippy_gateway
|
||||
- clippy_data
|
||||
- clippy_rest
|
||||
@@ -164,6 +311,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify clippy jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; clippy jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.clippy_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_rest.result }}" != "success" ]; then
|
||||
@@ -173,82 +324,100 @@ jobs:
|
||||
|
||||
test_gateway:
|
||||
name: Test (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# 构建指纹提到 job 级:mold RUSTFLAGS / 栈 / sccache 对 lib、bins、integration 三步保持一致,
|
||||
# 避免 step 级 env 漂移导致同 job 内 rustc 指纹不一致。
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
# 与 rust-toolchain.toml、fmt/clippy 钉在同一版本,避免浮动 stable 换指纹导致全量重编
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# mold RUSTFLAGS 只在本 job 生效:独立 cache key,避免与无 mold 的 job 互相污染指纹
|
||||
shared-key: rust-ci-gateway-test-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Setup mold
|
||||
uses: rui314/setup-mold@v1
|
||||
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test lib
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --lib
|
||||
|
||||
- name: Test bins
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --bins
|
||||
|
||||
# 只运行独立 integration targets;显式列出目标,避免 --tests 再次执行 lib/bin 测试。
|
||||
- name: Test integration targets
|
||||
run: >-
|
||||
cargo nextest run -p aether-gateway
|
||||
--test admin_unsigned_identity_headers
|
||||
--test architecture_guard
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
test_data:
|
||||
name: Test (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_REQUIRE_LOCAL_POSTGRES_TESTS: "true"
|
||||
run: cargo nextest run -p aether-data
|
||||
|
||||
- name: Show sccache stats
|
||||
@@ -260,29 +429,31 @@ jobs:
|
||||
|
||||
check_data_features:
|
||||
name: Check (Data Feature - ${{ matrix.feature }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
feature:
|
||||
- postgres
|
||||
- mysql
|
||||
- sqlite
|
||||
- all-drivers
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check selected data driver
|
||||
env:
|
||||
@@ -299,27 +470,31 @@ jobs:
|
||||
|
||||
test_rest:
|
||||
name: Test (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
@@ -336,31 +511,33 @@ jobs:
|
||||
|
||||
test_data_adapters:
|
||||
name: Test (Data Adapter - ${{ matrix.package }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
package:
|
||||
- aether-data-postgres
|
||||
- aether-data-mysql
|
||||
- aether-data-sqlite
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test adapter
|
||||
env:
|
||||
@@ -377,21 +554,28 @@ jobs:
|
||||
|
||||
check_integration_scenarios:
|
||||
name: Test (Integration Scenarios)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test scenario binaries and end-to-end suites
|
||||
env:
|
||||
@@ -410,6 +594,7 @@ jobs:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- test_gateway
|
||||
- test_data
|
||||
- check_data_features
|
||||
@@ -420,6 +605,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify test jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; test jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.test_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.test_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.check_data_features.result }}" != "success" ] || \
|
||||
@@ -430,42 +619,10 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
data_db_smoke_sqlite:
|
||||
name: Data DB Smoke (SQLite)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run SQLite data smoke tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: cargo test -p aether-data --all-features sqlite --lib
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_postgres:
|
||||
name: Data DB Smoke (Postgres)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
@@ -482,22 +639,24 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Add PostgreSQL server binaries to PATH
|
||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||
@@ -516,6 +675,20 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_provider_upstream_metadata_migration_preserves_json_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run Postgres payment callback regression tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_payment_callback --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres batch wallet deduction regression
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_bulk_wallet_adjustment_persists_actual_delta_and_skips_zero_ledger --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres API key lifecycle tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -535,112 +708,6 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run SQLite-to-Postgres import smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features sqlite_core_export_reads_migrated_database_rows --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_mysql:
|
||||
name: Data DB Smoke (MySQL)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
env:
|
||||
MYSQL_DATABASE: aether_test
|
||||
MYSQL_USER: aether
|
||||
MYSQL_PASSWORD: aether
|
||||
MYSQL_ROOT_PASSWORD: aether_root
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uaether -paether --silent"
|
||||
--health-interval=5s
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run MySQL migration smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_migrations_create_core_config_tables_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage write smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_write_repository_upserts_and_flushes_counters_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_read_repository_reads_usage_contract_views_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL provider catalog smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_provider_catalog_repository_round_trips_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL core export smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_wallet_read_repository_reads_wallet_contract_views --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet daily usage aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL stats aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
@@ -652,16 +719,21 @@ jobs:
|
||||
name: Data DB Smoke
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- data_db_smoke_sqlite
|
||||
- changes
|
||||
- data_db_smoke_postgres
|
||||
- data_db_smoke_mysql
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify database smoke jobs
|
||||
run: |
|
||||
if [ "${{ needs.data_db_smoke_sqlite.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_postgres.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_mysql.result }}" != "success" ]; then
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; database smoke jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||
echo "Data DB smoke failed"
|
||||
exit 1
|
||||
fi
|
||||
@@ -670,18 +742,40 @@ jobs:
|
||||
name: check
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- fmt
|
||||
- clippy
|
||||
- test
|
||||
- data_db_smoke
|
||||
- shell_security
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify required jobs
|
||||
run: |
|
||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||
[ "${{ needs.test.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
||||
# changes 失败或未产出 scope 时不允许直接放行,避免假绿。
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
rust="${{ needs.changes.outputs.rust }}"
|
||||
shell="${{ needs.changes.outputs.shell }}"
|
||||
|
||||
if [ "$rust" != "true" ] && [ "$shell" != "true" ]; then
|
||||
echo "No Rust or shell scope changed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$rust" = "true" ] && {
|
||||
[ "${{ needs.fmt.result }}" != "success" ] ||
|
||||
[ "${{ needs.clippy.result }}" != "success" ] ||
|
||||
[ "${{ needs.test.result }}" != "success" ] ||
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ];
|
||||
}; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$shell" = "true" ] && [ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -13,6 +13,10 @@
|
||||
.plans
|
||||
.playwright-mcp/
|
||||
|
||||
docs/architecture
|
||||
!docs/architecture/architecture-dark.svg
|
||||
!docs/architecture/architecture-light.svg
|
||||
|
||||
### Python ###
|
||||
*.db
|
||||
*.db-*
|
||||
|
||||
Generated
+624
-361
File diff suppressed because it is too large
Load Diff
+4
-5
@@ -11,8 +11,6 @@ members = [
|
||||
"crates/aether-routing-core",
|
||||
"crates/aether-data/contracts",
|
||||
"crates/aether-data/adapters/postgres",
|
||||
"crates/aether-data/adapters/mysql",
|
||||
"crates/aether-data/adapters/sqlite",
|
||||
"crates/aether-data/query",
|
||||
"crates/aether-data/schema",
|
||||
"crates/aether-dispatch-core",
|
||||
@@ -66,8 +64,6 @@ aether-provider-pool = { path = "crates/aether-provider/pool" }
|
||||
aether-routing-core = { path = "crates/aether-routing-core" }
|
||||
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
||||
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
||||
aether-data-mysql = { path = "crates/aether-data/adapters/mysql" }
|
||||
aether-data-sqlite = { path = "crates/aether-data/adapters/sqlite" }
|
||||
aether-data-query = { path = "crates/aether-data/query" }
|
||||
aether-data-schema = { path = "crates/aether-data/schema" }
|
||||
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
||||
@@ -96,11 +92,13 @@ aether-usage-core = { path = "crates/aether-usage/core" }
|
||||
aether-usage-runtime = { path = "crates/aether-usage/runtime" }
|
||||
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
|
||||
aether-gateway = { path = "apps/aether-gateway" }
|
||||
aether-tunnel = { path = "apps/aether-tunnel" }
|
||||
aether-http = { path = "crates/aether-http" }
|
||||
aether-runtime = { path = "crates/aether-runtime/base" }
|
||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
axum = "0.8"
|
||||
@@ -117,8 +115,9 @@ flate2 = "1"
|
||||
futures-util = "0.3"
|
||||
hmac = "0.12"
|
||||
http = "1"
|
||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
||||
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||
percent-encoding = "2"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||
regex = "1"
|
||||
|
||||
+9
-4
@@ -10,20 +10,24 @@
|
||||
|
||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||
# distroless has no shell, so we use busybox to set up the symlink.
|
||||
FROM busybox:1.37-musl AS layout
|
||||
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||
|
||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||
|
||||
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||
# identity can traverse and read every packaged asset.
|
||||
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
|
||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||
|
||||
# --- final stage: distroless runtime ---
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||
|
||||
COPY --from=layout /opt/aether /opt/aether
|
||||
|
||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
||||
|
||||
ENV RUST_LOG=aether_gateway=info \
|
||||
APP_PORT=8084 \
|
||||
HOME=/tmp/aether-home \
|
||||
AETHER_UPDATE_STRATEGY=docker \
|
||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||
|
||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER root
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -157,4 +157,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
||||
|
||||
@@ -156,4 +156,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER 0:0
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -6,7 +6,7 @@ DEV_RUST_LOG := $(RUST_LOG)
|
||||
endif
|
||||
export DEV_RUST_LOG
|
||||
|
||||
.PHONY: dev dev-backend dev-frontend migration backfill
|
||||
.PHONY: dev dev-backend dev-frontend db-status db-prepare migration backfill
|
||||
|
||||
define DEV_BACKEND_SCRIPT
|
||||
set -euo pipefail
|
||||
@@ -20,6 +20,13 @@ set -a
|
||||
source .env
|
||||
set +a
|
||||
|
||||
if [[ -n "$${ADMIN_EMAIL:-}" || -n "$${ADMIN_USERNAME:-}" || -n "$${ADMIN_PASSWORD:-}" ]]; then
|
||||
if [[ -z "$${ADMIN_USERNAME:-}" || -z "$${ADMIN_PASSWORD:-}" ]]; then
|
||||
echo "=> 管理员自举配置不完整,请在 .env 中设置 ADMIN_USERNAME 和 ADMIN_PASSWORD"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
dotenv_has_key() {
|
||||
local key="$$1"
|
||||
grep -Eq "^[[:space:]]*$${key}=" .env
|
||||
@@ -29,15 +36,6 @@ lowercase() {
|
||||
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
dev_uses_sqlite_database() {
|
||||
local driver
|
||||
local url
|
||||
driver="$$(lowercase "$${AETHER_DATABASE_DRIVER:-}")"
|
||||
url="$${AETHER_DATABASE_URL:-$${DATABASE_URL:-}}"
|
||||
|
||||
[[ "$${driver}" == "sqlite" || "$${url}" == sqlite:* ]]
|
||||
}
|
||||
|
||||
dev_uses_postgres_database() {
|
||||
local driver
|
||||
local url
|
||||
@@ -61,9 +59,6 @@ dev_uses_redis_runtime() {
|
||||
if [[ "$${backend}" == "redis" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if dev_uses_sqlite_database; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -201,12 +196,17 @@ print_startup_failure_hint() {
|
||||
|
||||
if [ -n "$${log_file}" ] && [ -f "$${log_file}" ]; then
|
||||
if grep -Eq "database schema is behind" "$${log_file}"; then
|
||||
echo "=> 检测到数据库 schema 落后,请执行: make migration"
|
||||
echo "=> 检测到数据库尚未准备完成,请执行: make db-prepare"
|
||||
return
|
||||
fi
|
||||
|
||||
if grep -Eq "database backfills are behind" "$${log_file}"; then
|
||||
echo "=> 检测到待执行 backfills,请执行: make backfill"
|
||||
echo "=> 检测到数据库尚未准备完成,请执行: make db-prepare"
|
||||
return
|
||||
fi
|
||||
|
||||
if grep -Eq "bootstrap admin env is partially configured.*ADMIN_PASSWORD" "$${log_file}"; then
|
||||
echo "=> 首次启动需要管理员密码,请在 .env 中设置 ADMIN_PASSWORD"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
@@ -344,6 +344,9 @@ if ! ensure_dev_infra; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=> 编译 aether-gateway..."
|
||||
cargo build -p aether-gateway --bin aether-gateway
|
||||
|
||||
GATEWAY_PID=""
|
||||
GATEWAY_LOG_DIR=""
|
||||
GATEWAY_LOG_FILE=""
|
||||
@@ -352,8 +355,8 @@ create_gateway_log_file
|
||||
|
||||
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
||||
echo "=> 日志过滤: $${RUST_LOG}"
|
||||
echo "=> 执行命令: cargo run -p aether-gateway -- --app-port $${APP_PORT}"
|
||||
cargo run -p aether-gateway -- --app-port "$${APP_PORT}" > >(
|
||||
echo "=> 执行命令: target/debug/aether-gateway --app-port $${APP_PORT}"
|
||||
target/debug/aether-gateway --app-port "$${APP_PORT}" > >(
|
||||
tee -a "$${GATEWAY_LOG_FILE}"
|
||||
) 2>&1 &
|
||||
GATEWAY_PID=$$!
|
||||
@@ -444,7 +447,7 @@ if [ -f .env ]; then
|
||||
fi
|
||||
export APP_PORT="$${APP_PORT:-8084}"
|
||||
|
||||
echo "=> 启动后端: RUST_LOG=$${DEV_RUST_LOG} cargo run -p aether-gateway -- --app-port $${APP_PORT:-8084}"
|
||||
echo "=> 启动后端: 先编译 aether-gateway,再运行 target/debug/aether-gateway --app-port $${APP_PORT:-8084}"
|
||||
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
||||
backend_pid=$$!
|
||||
|
||||
@@ -494,8 +497,14 @@ export DEV_SCRIPT
|
||||
define DB_TASK_SCRIPT
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "$${DB_TASK_FLAG:-}" ] || [ -z "$${DB_TASK_LABEL:-}" ]; then
|
||||
echo "=> 内部错误: DB_TASK_FLAG / DB_TASK_LABEL 未设置"
|
||||
if [ -z "$${DB_TASK_COMMAND:-}" ] || [ -z "$${DB_TASK_LABEL:-}" ]; then
|
||||
echo "=> 内部错误: DB_TASK_COMMAND / DB_TASK_LABEL 未设置"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -r -a db_task_args <<< "$${DB_TASK_COMMAND}"
|
||||
if [ "$${#db_task_args[@]}" -eq 0 ]; then
|
||||
echo "=> 内部错误: DB_TASK_COMMAND 为空"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -546,8 +555,8 @@ if ! command -v cargo >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=> 执行 $${DB_TASK_LABEL}: cargo run -p aether-gateway -- $${DB_TASK_FLAG}"
|
||||
exec cargo run -p aether-gateway -- "$${DB_TASK_FLAG}"
|
||||
echo "=> 执行 $${DB_TASK_LABEL}: cargo run -p aether-gateway --bin aether-gateway -- $${db_task_args[*]}"
|
||||
exec cargo run -p aether-gateway --bin aether-gateway -- "$${db_task_args[@]}"
|
||||
endef
|
||||
export DB_TASK_SCRIPT
|
||||
|
||||
@@ -560,8 +569,14 @@ dev-backend:
|
||||
dev-frontend:
|
||||
@cd frontend && npm run dev
|
||||
|
||||
db-status:
|
||||
@DB_TASK_COMMAND="db status" DB_TASK_LABEL="数据库状态检查" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
|
||||
db-prepare:
|
||||
@DB_TASK_COMMAND="db prepare" DB_TASK_LABEL="数据库准备" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
|
||||
migration:
|
||||
@DB_TASK_FLAG=--migrate DB_TASK_LABEL="数据库迁移" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
@DB_TASK_COMMAND="--migrate" DB_TASK_LABEL="数据库迁移" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
|
||||
backfill:
|
||||
@DB_TASK_FLAG=--apply-backfills DB_TASK_LABEL="数据库 backfill" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
@DB_TASK_COMMAND="--apply-backfills" DB_TASK_LABEL="数据库 backfill" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||
|
||||
@@ -44,90 +44,44 @@ cd Aether
|
||||
|
||||
# 2. 配置环境变量
|
||||
cp .env.example .env
|
||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
||||
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||
chmod 600 .env
|
||||
# 生成 JWT / 加密 / Postgres / Redis 独立随机密钥,并填入 .env
|
||||
./generate_keys.sh
|
||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||
|
||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
||||
# Postgres + Redis (适用于企业或多人使用)
|
||||
# 3. Docker 部署 / 更新(PostgreSQL + Redis)
|
||||
docker compose pull && docker compose up -d
|
||||
# Single Node (适用于个人用户或朋友分享)
|
||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
||||
```
|
||||
|
||||
### 一键更新
|
||||
|
||||
Docker Compose 部署后,可在部署目录直接执行:
|
||||
|
||||
```bash
|
||||
./update.sh
|
||||
```
|
||||
|
||||
`update.sh` 会拉取最新 `app` 镜像并重建 `app` 容器,Docker named volumes、`./data` 和 `./logs` 不会被删除。Single Node 部署也可显式指定:
|
||||
|
||||
```bash
|
||||
./update.sh --mode single-node
|
||||
```
|
||||
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
||||
|
||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
||||
|
||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
||||
|
||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
||||
|
||||
如果是本地源码构建镜像的部署,继续使用:
|
||||
|
||||
```bash
|
||||
./deploy.sh
|
||||
```
|
||||
|
||||
如果要在本机联调“管理后台在线更新”本身,可启动仓库内置的 release-layout 测试环境:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.release-local.yml up -d --build
|
||||
```
|
||||
|
||||
这套环境会用当前源码构建一个本地测试镜像,但编译为 `release` 类型,并默认伪装成 `v0.7.0`,这样后台会按正式发布版逻辑开放“立即更新”。默认监听 `http://127.0.0.1:18085`,数据目录使用 `./data-release-local`;日志默认走 `docker logs`,不会影响你正在跑的源码构建容器。
|
||||
|
||||
如果这套容器在 `prepare-update` 时访问 GitHub 失败,而你本机是通过代理出网,请在 `.env` 里把 `AETHER_UPDATE_PROXY_URL` 写成宿主机地址,例如 `http://host.docker.internal:7890`;容器内的 `127.0.0.1` 指向容器自身,不是宿主机。
|
||||
|
||||
如果想重置这套联调环境(包括 `/opt/aether/current` 和已下载的历史版本),执行:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.release-local.yml down -v
|
||||
```
|
||||
|
||||
可选变量:
|
||||
|
||||
- `AETHER_RELEASE_LOCAL_VERSION`:本地联调镜像对外声明的当前版本,默认 `v0.7.0`
|
||||
- `AETHER_RELEASE_LOCAL_PORT`:本地联调端口,默认 `18085`
|
||||
- `LOCAL_RELEASE_APP_IMAGE`:本地联调镜像名,默认 `aether-app:release-local`
|
||||
|
||||
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
||||
### 一键安装(PostgreSQL + Redis)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/fawney19/Aether.git
|
||||
cd Aether
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose
|
||||
```
|
||||
|
||||
正式版和 Nightly 自动构建仅提供 Linux `amd64` / `arm64` 二进制包,Docker 镜像同样支持这两种架构。macOS 用户可使用 Docker 或自行从源码构建;安装脚本保留对历史 macOS 制品的兼容。独立 Aether Tunnel 的多平台发行不受此调整影响。
|
||||
|
||||
原生 Linux systemd 安装需先准备 PostgreSQL,将连接串通过 `DATABASE_URL` 传给安装进程,并选择 `--mode single-node`;不再自动创建本地数据库文件。
|
||||
|
||||
### Nightly(每日 main 构建)
|
||||
|
||||
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
||||
|
||||
安装最新 nightly(Linux systemd / macOS launchd + SQLite):
|
||||
安装最新 nightly(PostgreSQL + Redis):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --channel nightly
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose --channel nightly
|
||||
```
|
||||
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制方式可重新执行上述安装命令升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制部署请沿用已有 PostgreSQL 环境配置,并使用 `--mode single-node --channel nightly` 重新运行安装脚本升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
|
||||
## 本地开发
|
||||
|
||||
依赖 Docker、Rust toolchain、Node.js 和 make。
|
||||
首次启动前需要在 `.env` 中设置 `ADMIN_PASSWORD`,用于创建本地管理员。
|
||||
|
||||
```bash
|
||||
make dev
|
||||
@@ -135,6 +89,12 @@ make dev
|
||||
|
||||
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
||||
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
||||
`make dev` 会先完成后端编译,再开始计算服务健康检查超时。数据库 schema 和必要的派生数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。升级不会主动重写或清除已有业务历史记录,新写入会直接遵循当前的数据持久化策略。排查或部署前预执行时可使用:
|
||||
|
||||
```bash
|
||||
make db-status
|
||||
make db-prepare
|
||||
```
|
||||
|
||||
## Codex 远程协同
|
||||
|
||||
@@ -161,25 +121,57 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
||||
## 环境变量
|
||||
|
||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||
- `DATABASE_URL`:数据库连接串;SQLite 例如 `sqlite:///opt/aether/data/aether.db`,Postgres 例如 `postgresql://postgres:aether@postgres:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS` / `AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS`:普通数据库连接的单条 SQL / 锁等待期限,默认 `30000` / `3000` 毫秒,显式 `0` 关闭;不是整个事务总期限。schema 迁移使用独立超时配置,历史 backfill 使用独立连接放宽期限
|
||||
- `AETHER_POSTGRES_MIGRATION_LOCK_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_CONCURRENT_TIMEOUT_MS`:schema 迁移的锁等待、每个事务及并发索引迁移期限,默认 `1000` / `10000` / `900000` 毫秒,不接受 `0`。超时会中止当前迁移,已提交的迁移保留;空库 schema 初始化也受事务期限约束
|
||||
- `AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES`:usage 诊断正文共享预算,默认 `134217728`(128 MiB),按 JSON 堆内存估算,覆盖进入终态队列的 seed、Redis 解码后的事件、数据库写入 DTO 及其正文副本。额度不足或显式 `0` 时先保留计费事实,再舍弃诊断正文;已有清空或禁用状态保持不变,其余标记截断。预算随正文保留到释放,后台构建或压缩不会因调用方取消而提前归还额度。该额度不覆盖原始 Redis 批次、解码临时分配、序列化及压缩结果、协议观察缓冲或进程总内存;可通过 `usage_runtime_event_capture_memory_*` 指标观察
|
||||
- `AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES`:新增 usage 队列消息的完整 JSON payload 上限,默认 `1048576`(1 MiB),按序列化后的 UTF-8 字节计算,显式 `0` 非法。超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义时拒绝入队,终态消息尝试受限数据库落库,失败则明确失败,不继续 Redis 重试。该限制不覆盖存量 Redis 消息、整个读取批次、DLQ 或进程总内存。`usage_runtime_queue_payload_*` 导出上限及进程级降级、拒绝编码尝试次数,包含入队和重试预校验,不代表唯一事件数;`usage_runtime_enqueue_retry_permanent_failure_total` 记录永久输入错误导致的重试拒绝或终止
|
||||
- `AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES` / `AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES`:usage worker 读取和重领共用的进程级逻辑 payload 预留,默认总额 `134217728`(128 MiB)、单批目标 `8388608`(8 MiB)。按当前 `QUEUE_PAYLOAD_MAX_BYTES` 推导实际 COUNT,默认最多读取 8 条,自动扩容使用实际 COUNT 判断批次是否读满。预留覆盖读取、整批处理和确认,额度不足等待;取消/失败释放。单批目标至少允许一条,当前 payload 上限大于总额时读取报配置错误。`0` 或非法值回退默认,过大值收敛到约 4 GiB 的有效总额。收到消息后按全部字段值长度缩减多余预留;历史消息、其他生产者使用更高上限或额外字段可能超出估算,仍继续原计费流程并记录 `usage_runtime_queue_read_oversized_*`。`usage_runtime_queue_read_*` 同时导出预留、等待与累计字段字节;该预留不是 RESP 解码、连接缓冲容量、字段结构、诊断 JSON、DLQ 或进程 RSS 的硬上限,旧公开 Vec 读取接口不携带处理阶段预留
|
||||
- `AETHER_USAGE_DLQ_ENCODING_BUDGET_BYTES` / `AETHER_USAGE_DLQ_ENCODING_MAX_JOBS`:死信原文和 JSON 编码独立共享预留,默认 `67108864`(64 MiB)、最多 `4` 个后台编码及写入任务。根据原始字段、ID、错误字符串及 JSON 最坏 6 倍转义一次预留;预算占满或单条超总额时立即失败,worker 保留原消息等待重领,不截断账务原文。编码失败会继续处理同批其他消息,只确认成功项,批次末尾仍报告失败;存储转移失败则停止该批后续处理。取消编码等待不会提前归还仍在后台使用的额度。`0`/非法值回退默认,bytes 最大约 4 GiB,jobs 最大 128;超大存量消息可能需要调高总额后恢复。`usage_runtime_dlq_encoding_*` 导出额度、在途任务、拒绝和编码尝试次数;不包含字段结构、字符串额外容量、Redis 命令/连接副本或进程 RSS。内置 Redis/Memory worker 将死信追加、源 ACK 和删除作为一次原子转移,同一源 stream、消费组及 pending ID 的并发或重试只追加一次;Redis 要求 7+ 及 `EVAL/TYPE/XPENDING/XADD/XACK/XDEL` 权限,Cluster 两键须同 slot(当前默认键不自动迁移)。源和 DLQ 不能同名。源已不在 PEL 时不宣称已归档;外部 ACK/trim/delete 及多消费组仍有原来的删除语义。公开 `push_dead_letter` 仍为追加接口,未实现新原子 trait 方法的外部后端沿用追加后 ACK,仍可能重复归档
|
||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_GATEWAY_MAX_HTTP_CONNECTIONS`:二进制入口全部监听分片共用的入站 TCP 连接上限,包含握手、空闲 keep-alive 和 HTTP 升级后仍存活的 socket。未设置或 `0` 时使用请求上限与 WebSocket 上限之和;自动及显式值均最多 `65536`,已知 FD soft limit 时进一步限制为 `max(1, (FD - 256) / 2)`。接入后立即尝试取得额度,满额时关闭新连接,不创建 HTTP 处理任务、不等待额度,不返回 HTTP 状态码;取消、解析失败和连接释放归还,WebSocket 升级不会提前归还。HTTP/2 多流共用一个 TCP 许可,原请求和 WebSocket 准入仍独立有效。`gateway_http_connections_*` 导出配置上限、当前数、高水位、拒绝数及 accept 错误数。该限制不包含 kernel backlog、上游、Redis 或数据库连接,也不是整个进程 FD/内存硬上限。临时 accept 错误重试,资源类错误退避一秒后重试,避免单次错误停止监听
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`;压缩和未知长度上传按实际缓冲增长申请额度,解压时计入同时存活的输入和输出。额度不足返回 `503`;接近单请求上限的压缩上传需要为输入和解压输出预留额外预算
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`;显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||
- `AETHER_GATEWAY_UPSTREAM_STREAM_IDLE_TIMEOUT_MS`:上游流首包后的空闲超时,默认 `300000ms`;请求执行配置中的 `read_ms` 优先,显式 `0` 关闭对应超时。网关生成的 keepalive 不会重置计时
|
||||
- `AETHER_GATEWAY_STREAM_CAPTURE_MEMORY_BUDGET_BYTES`:进程内流式响应诊断捕获的共享字节预算,默认 `134217728`(128 MiB);包含 provider/client 捕获容量和扩容时的新旧分配。额度不足时仅截断审计副本,显式 `0` 关闭此类捕获;协议解析、客户端传输和计费观察继续执行。该预算不包含协议解析缓冲、终态编码及 usage 队列副本,不是进程总内存上限
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||
- `AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS`:跨网关 owner relay 解析到私有/保留地址时的显式运维开关,默认关闭;仅当多网关 relay URL 是受控的内网 HTTPS 地址时设置为 `true`。它不改变普通 provider 请求的 DNS/代理策略,也不允许明文 HTTP 非 loopback relay
|
||||
- `AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST`:更窄的 owner relay 私网例外,填写逗号分隔的精确主机名(例如 `gateway-a.internal,gateway-b.internal`,忽略大小写和末尾点);仅这些主机解析出的私有地址会被允许,并且请求仍使用解析后地址 pin。不要填写通配符或 `.internal` 这类后缀
|
||||
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:常规启动前自动执行挂起的 schema migration 和 backfill;仓库自带的 `docker-compose.yml` 默认开启
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。配置 Redis 时使用 `redis`,否则使用 `memory`;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
||||
- `DB_PASSWORD` / `REDIS_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||
|
||||
运行日志由独立后台线程写入 stdout 和文件,每个输出队列最多 4096 条、保留正文最多 8 MiB(包含正在写入的记录),单条最多 256 KiB。队列满、正文预算不足或单条超限时整条丢弃,不等待日志设备;`Both` 两个输出独立降级。`logging_stdout_*` 和 `logging_file_*` 指标记录丢弃和写入错误,网关指标沿用其命名空间前缀。正常退出时日志最多等待 2 秒排空;这不是请求优雅排空或整个进程退出期限。日志格式化仍在调用线程执行,日志预算不包含格式化临时内存,运行日志也不能作为可靠计费账本。
|
||||
|
||||
### S3 备份离线恢复
|
||||
|
||||
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||
|
||||
```bash
|
||||
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||
--input ./backup.json.zst.aes256gcm \
|
||||
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||
--output ./restored-backup.json
|
||||
```
|
||||
|
||||
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||
|
||||
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ npm --prefix aether-vscodex/web run build
|
||||
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
||||
共享连接目录前横向扩容 sidecar。
|
||||
|
||||
登录 Aether 后打开“Codex 远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
登录 Aether 后打开“远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
||||
VS Code SecretStorage,并同时保持本机控制台连接。
|
||||
|
||||
|
||||
@@ -62,17 +62,18 @@ flate2.workspace = true
|
||||
futures-util.workspace = true
|
||||
hmac.workspace = true
|
||||
http.workspace = true
|
||||
http-body = "1"
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
||||
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||
libc = "0.2"
|
||||
md-5 = "0.10"
|
||||
object_store.workspace = true
|
||||
parking_lot = "0.12"
|
||||
percent-encoding.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
rsa = "0.9.10"
|
||||
rustls.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -81,7 +82,7 @@ sha1 = "0.10"
|
||||
sha2 = { workspace = true, features = ["oid"] }
|
||||
socket2.workspace = true
|
||||
tar.workspace = true
|
||||
sqlx = { workspace = true, features = ["postgres", "mysql", "sqlite", "migrate"] }
|
||||
sqlx = { workspace = true, features = ["postgres", "migrate"] }
|
||||
sysinfo = "0.32"
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -102,5 +103,7 @@ tikv-jemallocator = { version = "0.6", optional = true }
|
||||
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aether-testkit = { workspace = true, features = ["postgres"] }
|
||||
aether-test-support.workspace = true
|
||||
aws-lc-rs.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
@@ -30,7 +30,7 @@ struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||
default_value = "/tmp/aether-execution-runtime.sock"
|
||||
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||
)]
|
||||
unix_socket: PathBuf,
|
||||
|
||||
@@ -71,8 +71,13 @@ struct Args {
|
||||
distributed_request_command_timeout_ms: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||
run()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
init_service_runtime(ServiceRuntimeConfig::new(
|
||||
|
||||
@@ -88,8 +88,13 @@ struct Args {
|
||||
distributed_request_command_timeout_ms: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _log_shutdown = aether_runtime::LogShutdownGuard::new();
|
||||
run()
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
init_service_runtime(ServiceRuntimeConfig::new(
|
||||
"aether-tunnel-standalone",
|
||||
"aether_gateway=info",
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
pub(crate) use crate::handlers::admin::{
|
||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
||||
reconcile_admin_fixed_provider_template_endpoints,
|
||||
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
||||
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
||||
OAUTH_REQUEST_FAILED_PREFIX,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||
};
|
||||
|
||||
use crate::handlers::admin::{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||
use base64::Engine as _;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
||||
}
|
||||
|
||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let Some(normalized_bytes) =
|
||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||
else {
|
||||
|
||||
@@ -69,9 +69,14 @@ pub(crate) use aether_ai_formats::api::{
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::protocol::stream::CanonicalUsage as StreamingCanonicalUsage;
|
||||
/// Codex client identity headers re-exported for out-of-crate probe binaries,
|
||||
/// which must reach `aether_ai_formats` through this seam.
|
||||
pub use aether_ai_formats::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
/// Codex client identity accessors re-exported for out-of-crate probe binaries,
|
||||
/// which must reach the runtime profile through this seam.
|
||||
pub use aether_ai_formats::{codex_client_originator, codex_client_user_agent};
|
||||
/// Codex 动态客户端画像 API 只允许经此根缝进入 gateway,避免其它模块直接依赖 formats crate。
|
||||
pub(crate) use aether_ai_formats::{
|
||||
codex_client_profile, codex_client_version, set_codex_cli_version, set_codex_client_profile,
|
||||
CodexClientProfile,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::{CODEX_RESPONSES_LITE_HEADER, UPSTREAM_IS_STREAM_KEY};
|
||||
|
||||
pub(crate) fn parse_direct_request_body(
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_provider_transport::CodexFingerprintConvergenceContext;
|
||||
use http::{request::Parts, HeaderMap};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::client_session_affinity::codex_request_signals_from_request;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<CodexFingerprintConvergenceContext>>);
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<ProviderOutboundRequestContext>>);
|
||||
|
||||
impl Default for CodexFingerprintContextSlot {
|
||||
fn default() -> Self {
|
||||
@@ -18,11 +18,7 @@ impl Default for CodexFingerprintContextSlot {
|
||||
}
|
||||
|
||||
impl CodexFingerprintContextSlot {
|
||||
fn resolve(
|
||||
&self,
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
fn resolve(&self, headers: &HeaderMap, body_json: &Value) -> ProviderOutboundRequestContext {
|
||||
self.0
|
||||
.get_or_init(|| {
|
||||
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
||||
@@ -34,10 +30,10 @@ impl CodexFingerprintContextSlot {
|
||||
pub(crate) fn resolve_codex_fingerprint_context(
|
||||
parts: &Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
if let Some(context) = parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.cloned()
|
||||
{
|
||||
return context;
|
||||
@@ -51,7 +47,7 @@ pub(crate) fn resolve_codex_fingerprint_context(
|
||||
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
&& parts
|
||||
.extensions
|
||||
@@ -67,11 +63,11 @@ pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
pub(crate) fn ensure_codex_fingerprint_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context = resolve_codex_fingerprint_context(parts, body_json);
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
{
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -84,7 +80,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
logical_turn_id: &str,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context =
|
||||
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -94,7 +90,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
|
||||
pub(crate) fn restore_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
context: &CodexFingerprintConvergenceContext,
|
||||
context: &ProviderOutboundRequestContext,
|
||||
) {
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
parts.extensions.insert(context.clone());
|
||||
@@ -104,10 +100,9 @@ fn build_codex_fingerprint_context(
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
logical_turn_id: String,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
||||
let mut context =
|
||||
CodexFingerprintConvergenceContext::new(logical_turn_id, current_unix_millis());
|
||||
let mut context = ProviderOutboundRequestContext::new(logical_turn_id, current_unix_millis());
|
||||
|
||||
if let Some(turn_id) = signals.turn_id {
|
||||
context = context.with_original_turn_id(turn_id);
|
||||
@@ -160,14 +155,14 @@ mod tests {
|
||||
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
||||
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
||||
assert_eq!(
|
||||
parts.extensions.get::<CodexFingerprintConvergenceContext>(),
|
||||
parts.extensions.get::<ProviderOutboundRequestContext>(),
|
||||
Some(&context)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_context_wins_over_retry_request_signals() {
|
||||
let original = CodexFingerprintConvergenceContext::new("logical-turn", 1234)
|
||||
let original = ProviderOutboundRequestContext::new("logical-turn", 1234)
|
||||
.with_original_turn_id("original-turn")
|
||||
.with_original_client_session_id("original-thread")
|
||||
.with_original_prompt_cache_key("original-cache");
|
||||
|
||||
@@ -18,6 +18,12 @@ pub(crate) fn maybe_build_local_stream_rewriter<'a>(
|
||||
}
|
||||
|
||||
impl LocalStreamRewriter<'_> {
|
||||
pub(crate) fn into_owned(self) -> LocalStreamRewriter<'static> {
|
||||
LocalStreamRewriter {
|
||||
inner: self.inner.into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_chunk(&mut self, chunk: &[u8]) -> Result<Vec<u8>, GatewayError> {
|
||||
self.inner.push_chunk(chunk).map_err(map_surface_error)
|
||||
}
|
||||
|
||||
@@ -11,10 +11,10 @@ use super::{
|
||||
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
||||
maybe_build_local_core_sync_finalize_response,
|
||||
};
|
||||
use crate::ai_serving::GatewayControlDecision;
|
||||
use crate::ai_serving::{
|
||||
convert_openai_chat_response_to_openai_responses,
|
||||
convert_openai_responses_response_to_openai_chat,
|
||||
convert_openai_responses_response_to_openai_chat, openai_responses_message_item_id,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::usage::GatewaySyncReportRequest;
|
||||
|
||||
@@ -192,7 +192,7 @@ fn aggregates_openai_responses_stream_completed_event_to_final_response() {
|
||||
"output_text": "Hello",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -843,7 +843,7 @@ fn converts_claude_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Claude CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -907,7 +907,7 @@ fn converts_claude_cli_tool_use_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -977,7 +977,7 @@ fn converts_gemini_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Gemini CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1046,7 +1046,7 @@ fn converts_gemini_cli_function_call_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "resp_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1252,7 +1252,7 @@ fn local_finalize_handles_openai_responses_openai_family_sync_response_even_when
|
||||
"model": "gpt-5",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_family_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_family_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
|
||||
@@ -9,7 +9,7 @@ use aether_ai_serving::{
|
||||
use aether_dispatch_core::{DispatchSequence, DispatchSequenceItem};
|
||||
use aether_routing_core::{
|
||||
rank_vector_for_candidate, CandidateKind, ResolvedRoutingPolicy, RoutingCandidateFacts,
|
||||
RoutingCandidateTrace, RoutingDecisionTrace,
|
||||
RoutingCandidateTrace, RoutingDecisionTrace, RoutingExecutionPolicy,
|
||||
};
|
||||
use aether_scheduler_core::{
|
||||
ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate, SchedulerRankingOutcome,
|
||||
@@ -79,6 +79,13 @@ type DecorateSkippedCandidateFn<'a> = Arc<
|
||||
pub(crate) trait LocalExecutionAttemptSource<T>: Send {
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<T>, GatewayError>;
|
||||
|
||||
/// Returns the request-scoped execution behaviour selected by routing.
|
||||
/// Execution wrappers use this snapshot before consuming the first
|
||||
/// attempt, avoiding a second lookup against mutable system settings.
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
None
|
||||
}
|
||||
|
||||
async fn drain_execution_attempts(&mut self) -> Result<Vec<T>, GatewayError>;
|
||||
|
||||
async fn skip_credential(&mut self, key_id: &str) -> Result<(), GatewayError>;
|
||||
@@ -989,7 +996,7 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
||||
);
|
||||
|
||||
if page_is_exact_auth_api_key_concurrency_limited(&page) {
|
||||
if self.wait_for_auth_api_key_concurrency_retry().await {
|
||||
if self.wait_for_auth_api_key_concurrency_retry().await? {
|
||||
continue;
|
||||
}
|
||||
self.persist_final_auth_api_key_concurrency_skips(page.skipped_candidates)
|
||||
@@ -1080,20 +1087,23 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> bool {
|
||||
async fn wait_for_auth_api_key_concurrency_retry(&mut self) -> Result<bool, GatewayError> {
|
||||
let now = Instant::now();
|
||||
let deadline = *self
|
||||
.auth_api_key_concurrency_wait_deadline
|
||||
.get_or_insert(now + AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||
if now >= deadline {
|
||||
return false;
|
||||
if !crate::scheduler::candidate::wait_for_auth_api_key_concurrency_retry(
|
||||
self.state.app(),
|
||||
Some(&self.auth_snapshot),
|
||||
deadline,
|
||||
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let sleep_duration =
|
||||
AUTH_API_KEY_CONCURRENCY_RETRY_DELAY.min(deadline.saturating_duration_since(now));
|
||||
tokio::time::sleep(sleep_duration).await;
|
||||
self.page_cursor.restart_scan();
|
||||
true
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn persist_final_auth_api_key_concurrency_skips(
|
||||
@@ -1237,9 +1247,7 @@ async fn scheduler_cache_affinity_enabled(
|
||||
state: PlannerAppState<'_>,
|
||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||
) -> bool {
|
||||
scheduler_ordering_config_for_routing_policy(state, routing_policy)
|
||||
.await
|
||||
.scheduling_mode
|
||||
scheduler_ordering_config_for_routing_policy(routing_policy).scheduling_mode
|
||||
== SchedulerSchedulingMode::CacheAffinity
|
||||
}
|
||||
|
||||
@@ -2284,6 +2292,103 @@ mod tests {
|
||||
candidate
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auth_concurrency_wait_paged_scan_retries_once_at_original_deadline() {
|
||||
let now = current_unix_ms();
|
||||
let active = serde_json::from_value(json!({
|
||||
"id": "active-candidate",
|
||||
"request_id": "active-request",
|
||||
"api_key_id": "api-key-1",
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"status": "pending",
|
||||
"is_cached": false,
|
||||
"created_at_unix_ms": now,
|
||||
"started_at_unix_ms": now
|
||||
}))
|
||||
.expect("active candidate should build");
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::seed([active]));
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_request_candidate_repository_for_tests(repository),
|
||||
);
|
||||
let mut auth_snapshot = sample_auth_snapshot();
|
||||
auth_snapshot.api_key_concurrent_limit = Some(1);
|
||||
let page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&crate::system_features::ModelDirectivePolicySnapshot::default(),
|
||||
"openai:chat",
|
||||
"gpt-5",
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModel,
|
||||
true,
|
||||
Some("trace-auth-wait"),
|
||||
)
|
||||
.await;
|
||||
let mut cursor = RequestedModelAttemptPageCursor {
|
||||
state: PlannerAppState::new(&app),
|
||||
trace_id: "trace-auth-wait".to_string(),
|
||||
client_api_format: "openai:chat".to_string(),
|
||||
requested_model: "gpt-5".to_string(),
|
||||
auth_snapshot,
|
||||
client_session_affinity: None,
|
||||
required_capabilities: None,
|
||||
routing_policy: None,
|
||||
sticky_session_token: None,
|
||||
request_auth_channel: None,
|
||||
skipped_user_id: "user-1".to_string(),
|
||||
skipped_api_key_id: "api-key-1".to_string(),
|
||||
skipped_required_capabilities: None,
|
||||
skipped_error_context: "test auth wait",
|
||||
record_runtime_miss_diagnostic: false,
|
||||
resolution_mode: LocalCandidateResolutionMode::Standard,
|
||||
decorate_skipped_candidate: Arc::new(identity_skipped_candidate),
|
||||
page_cursor,
|
||||
pending_items: VecDeque::new(),
|
||||
skipped_provider_ids: BTreeSet::new(),
|
||||
skipped_endpoint_ids: BTreeSet::new(),
|
||||
skipped_credential_ids: BTreeSet::new(),
|
||||
candidate_count: 0,
|
||||
next_candidate_index: 0,
|
||||
remembered_affinity: false,
|
||||
scheduler_cache_affinity_enabled: false,
|
||||
auth_api_key_concurrency_wait_deadline: None,
|
||||
deferred_error: None,
|
||||
};
|
||||
|
||||
let started = Instant::now();
|
||||
let mut scan_restarts = 0;
|
||||
while cursor
|
||||
.wait_for_auth_api_key_concurrency_retry()
|
||||
.await
|
||||
.expect("auth wait should succeed")
|
||||
{
|
||||
scan_restarts += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
scan_restarts, 1,
|
||||
"blocked polls must not restart page scans"
|
||||
);
|
||||
assert!(started.elapsed() >= AUTH_API_KEY_CONCURRENCY_WAIT_BUDGET);
|
||||
let original_deadline = cursor.auth_api_key_concurrency_wait_deadline;
|
||||
assert!(!cursor
|
||||
.wait_for_auth_api_key_concurrency_retry()
|
||||
.await
|
||||
.expect("expired auth wait should succeed"));
|
||||
assert_eq!(
|
||||
cursor.auth_api_key_concurrency_wait_deadline,
|
||||
original_deadline
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_group_keys_are_not_persisted_as_available_before_attempt() {
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
@@ -2320,16 +2425,11 @@ mod tests {
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[0].candidate_index, 2);
|
||||
assert_eq!(
|
||||
stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2477,14 +2577,25 @@ mod tests {
|
||||
|
||||
assert!(should_cache_resolved_candidate_page(&cursor));
|
||||
|
||||
let fixed_order_app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"scheduling_mode".to_string(),
|
||||
json!("fixed_order"),
|
||||
)]),
|
||||
);
|
||||
let fixed_order_app = AppState::new().expect("state should build");
|
||||
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-fixed-order".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
requested_model: "gpt-5".to_string(),
|
||||
resolved_model: "gpt-5".to_string(),
|
||||
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
matched_rules: Vec::new(),
|
||||
};
|
||||
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&fixed_order_app),
|
||||
&model_directive_policy,
|
||||
@@ -2494,7 +2605,7 @@ mod tests {
|
||||
true,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
Some(&fixed_order_policy),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
@@ -2512,7 +2623,7 @@ mod tests {
|
||||
auth_snapshot,
|
||||
client_session_affinity: None,
|
||||
required_capabilities: None,
|
||||
routing_policy: None,
|
||||
routing_policy: Some(fixed_order_policy),
|
||||
sticky_session_token: None,
|
||||
request_auth_channel: None,
|
||||
skipped_user_id: "user-1".to_string(),
|
||||
@@ -2610,16 +2721,11 @@ mod tests {
|
||||
);
|
||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[1].candidate_index, 1);
|
||||
assert_eq!(
|
||||
stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2689,7 +2795,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("ranking metadata should persist as object extra data");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
@@ -2702,14 +2808,7 @@ mod tests {
|
||||
Some(&json!("cached_affinity"))
|
||||
);
|
||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||
assert_eq!(
|
||||
extra_data
|
||||
.get("dispatch_ref")
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("ranked-key"))
|
||||
);
|
||||
assert!(extra_data.get("dispatch_ref").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3047,7 +3146,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("skipped ranking metadata should persist");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
|
||||
@@ -278,13 +278,21 @@ mod tests {
|
||||
|
||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||
Value::Bool(true)
|
||||
|
||||
@@ -6,14 +6,11 @@ use aether_ai_serving::{
|
||||
use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::ai_serving::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
||||
use crate::clock::current_unix_ms;
|
||||
use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
|
||||
use crate::scheduler::config::{
|
||||
read_scheduler_ordering_config, SchedulerOrderingConfig, SchedulerSchedulingMode,
|
||||
};
|
||||
use crate::scheduler::config::{SchedulerOrderingConfig, SchedulerSchedulingMode};
|
||||
use aether_scheduler_core::{
|
||||
matches_affinity_target, ClientSessionAffinity, SchedulerAffinityTarget,
|
||||
SchedulerMinimalCandidateSelectionCandidate, SchedulerPriorityMode, SchedulerRankableCandidate,
|
||||
@@ -133,7 +130,7 @@ pub(crate) async fn rank_eligible_local_execution_candidates(
|
||||
required_capabilities: Option<&serde_json::Value>,
|
||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||
) -> Vec<EligibleLocalExecutionCandidate> {
|
||||
let ordering_config = scheduler_ordering_config_for_routing_policy(state, routing_policy).await;
|
||||
let ordering_config = scheduler_ordering_config_for_routing_policy(routing_policy);
|
||||
let port = GatewayLocalCandidateRankingPort {
|
||||
state,
|
||||
requested_model,
|
||||
@@ -184,16 +181,24 @@ fn ai_ranking_scheduling_mode(mode: SchedulerSchedulingMode) -> AiRankingSchedul
|
||||
}
|
||||
}
|
||||
|
||||
/// Ordering config for a request. A resolved routing policy is authoritative
|
||||
/// and is never merged with legacy system-config values; without a policy the
|
||||
/// effective default (system-default routing group, then legacy keys) applies.
|
||||
pub(crate) async fn scheduler_ordering_config_for_routing_policy(
|
||||
state: PlannerAppState<'_>,
|
||||
/// Return the immutable scheduler snapshot carried by a resolved routing
|
||||
/// policy. A missing policy is a programming error in production request
|
||||
/// paths; unit tests may use the scheduler default for isolated ranking tests.
|
||||
pub(crate) fn scheduler_ordering_config_for_routing_policy(
|
||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||
) -> SchedulerOrderingConfig {
|
||||
match routing_policy {
|
||||
Some(policy) => SchedulerOrderingConfig::from_routing_policy(policy),
|
||||
None => read_scheduler_ordering_config_or_default(state).await,
|
||||
None => {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SchedulerOrderingConfig::default()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
panic!("resolved routing policy is required before candidate scheduling")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,23 +243,6 @@ fn routing_overlaid_candidate(
|
||||
overlaid
|
||||
}
|
||||
|
||||
async fn read_scheduler_ordering_config_or_default(
|
||||
state: PlannerAppState<'_>,
|
||||
) -> SchedulerOrderingConfig {
|
||||
match read_scheduler_ordering_config(state.app()).await {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
event_name = "planner_scheduler_ordering_config_load_failed",
|
||||
log_type = "event",
|
||||
error = ?error,
|
||||
"failed to load scheduler ordering config while ranking local execution candidates"
|
||||
);
|
||||
SchedulerOrderingConfig::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
@@ -263,10 +251,17 @@ mod tests {
|
||||
use aether_ai_serving::{
|
||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||
};
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::{
|
||||
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||
routing_profiles::InMemoryRoutingGroupRepository,
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::routing_profiles::{
|
||||
CreateRoutingGroupRecord, RoutingGroupWriteRepository,
|
||||
};
|
||||
use aether_scheduler_core::{
|
||||
apply_scheduler_candidate_ranking,
|
||||
build_scheduler_affinity_cache_key_for_api_key_id_with_client_session,
|
||||
@@ -296,7 +291,11 @@ mod tests {
|
||||
required_capabilities: Option<&serde_json::Value>,
|
||||
) -> Vec<SchedulerMinimalCandidateSelectionCandidate> {
|
||||
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
|
||||
let ordering_config = super::read_scheduler_ordering_config_or_default(state).await;
|
||||
let ordering_config =
|
||||
crate::scheduler::config::read_system_default_routing_ordering_config(state.app())
|
||||
.await
|
||||
.expect("routing strategy should load")
|
||||
.unwrap_or_default();
|
||||
let mut candidates = candidates;
|
||||
let mut rankables = Vec::with_capacity(candidates.len());
|
||||
let mut ordering_cache = CandidateTransportRankingFactsCache::default();
|
||||
@@ -363,6 +362,8 @@ mod tests {
|
||||
candidate.key_internal_priority = 3;
|
||||
candidate.key_global_priority_for_format = Some(2);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
@@ -372,6 +373,7 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: aether_routing_core::RankingOverlay::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: BTreeMap::new(),
|
||||
@@ -399,6 +401,8 @@ mod tests {
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
@@ -408,17 +412,14 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
matched_rules: Vec::new(),
|
||||
};
|
||||
|
||||
let ordering = super::scheduler_ordering_config_for_routing_policy(
|
||||
PlannerAppState::new(&state),
|
||||
Some(&policy),
|
||||
)
|
||||
.await;
|
||||
let ordering = super::scheduler_ordering_config_for_routing_policy(Some(&policy));
|
||||
|
||||
assert_eq!(
|
||||
ordering.scheduling_mode,
|
||||
@@ -437,6 +438,8 @@ mod tests {
|
||||
candidate.key_internal_priority = 3;
|
||||
candidate.key_global_priority_for_format = Some(2);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
@@ -446,6 +449,7 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: aether_routing_core::RankingOverlay {
|
||||
pool_priority_overrides: BTreeMap::from([("provider-1".to_string(), 4)]),
|
||||
key_priority_overrides: BTreeMap::from([("representative-key".to_string(), 1)]),
|
||||
@@ -569,6 +573,15 @@ mod tests {
|
||||
api_formats: Option<serde_json::Value>,
|
||||
allowed_models: Option<serde_json::Value>,
|
||||
) -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -580,7 +593,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
api_formats,
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:chat": 1})),
|
||||
@@ -694,7 +707,7 @@ mod tests {
|
||||
let observed_at_unix_secs = current_unix_secs();
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
@@ -703,6 +716,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-b".to_string(),
|
||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||
tunnel_generation: "test-generation-remote".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -713,6 +727,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-a".to_string(),
|
||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||
tunnel_generation: "test-generation-local".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -771,7 +786,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -824,7 +839,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"scheduling_mode".to_string(),
|
||||
@@ -881,7 +896,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -917,7 +932,8 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_execution_ranking_keeps_cross_format_priority_when_global_override_is_enabled() {
|
||||
async fn local_execution_ranking_keeps_cross_format_priority_when_strategy_override_is_enabled()
|
||||
{
|
||||
let provider_catalog = InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![
|
||||
sample_provider_with_options("provider-same", false, 10),
|
||||
@@ -932,14 +948,32 @@ mod tests {
|
||||
sample_key_for_provider("provider-cross", "key-cross", ""),
|
||||
],
|
||||
);
|
||||
let routing_repository = std::sync::Arc::new(InMemoryRoutingGroupRepository::default());
|
||||
routing_repository
|
||||
.create_routing_group(CreateRoutingGroupRecord {
|
||||
id: "strategy-default".to_string(),
|
||||
name: "strategy-default".to_string(),
|
||||
description: None,
|
||||
enabled: true,
|
||||
is_system_default: true,
|
||||
sort_order: 0,
|
||||
config_json: json!({
|
||||
"default_policy": {
|
||||
"keep_priority_on_conversion": true
|
||||
}
|
||||
}),
|
||||
version: 1,
|
||||
created_at: 1,
|
||||
updated_at: 1,
|
||||
published_at: None,
|
||||
})
|
||||
.await
|
||||
.expect("routing strategy should be created");
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"keep_priority_on_conversion".to_string(),
|
||||
json!(true),
|
||||
)]);
|
||||
.with_routing_group_repository_for_tests(routing_repository);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
@@ -999,7 +1033,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"provider_priority_mode".to_string(),
|
||||
@@ -1065,7 +1099,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1118,7 +1152,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1192,7 +1226,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1272,7 +1306,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1348,7 +1382,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1415,7 +1449,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1498,7 +1532,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1563,7 +1597,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1652,7 +1686,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1738,7 +1772,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1835,7 +1869,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1940,7 +1974,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -2034,7 +2068,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
|
||||
@@ -93,6 +93,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
&self,
|
||||
candidate: Self::Candidate,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason = "transport_snapshot_missing",
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason: "transport_snapshot_missing",
|
||||
@@ -145,6 +155,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
transport: Self::Transport,
|
||||
skip_reason: &'static str,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason,
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason,
|
||||
@@ -384,8 +404,7 @@ async fn resolve_and_rank_local_execution_candidates_with_pool_expansion(
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
scheduler_ordering_config_for_routing_policy(state, routing_policy)
|
||||
.await
|
||||
scheduler_ordering_config_for_routing_policy(routing_policy)
|
||||
.sticky_key_attempts,
|
||||
)
|
||||
};
|
||||
|
||||
@@ -6,7 +6,7 @@ use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use aether_runtime::ConcurrencyPermit;
|
||||
use aether_scheduler_core::{
|
||||
enumerate_minimal_candidate_selection_with_model_directives, normalize_api_format,
|
||||
resolve_requested_global_model_name_with_model_directives_and_request_operation,
|
||||
resolve_requested_global_model_name_with_reserved_global_model,
|
||||
row_supports_requested_model_with_model_directives_and_request_operation,
|
||||
ClientSessionAffinity, EnumerateMinimalCandidateSelectionInput,
|
||||
SchedulerMinimalCandidateSelectionCandidate,
|
||||
@@ -174,8 +174,9 @@ impl AiCandidatePreselectionPort for GatewayLocalCandidatePreselectionPort<'_> {
|
||||
self.ranking_seed,
|
||||
false,
|
||||
self.request_operation,
|
||||
self.routing_policy
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
self.routing_policy,
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -377,6 +378,7 @@ pub(crate) struct LocalCandidatePreselectionPageCursor<'a> {
|
||||
requested_name_offsets: BTreeMap<String, u32>,
|
||||
scanned_rows_by_format: BTreeMap<String, u32>,
|
||||
resolved_global_model_names: BTreeMap<String, String>,
|
||||
reserved_global_model_names: BTreeMap<String, Option<String>>,
|
||||
fallback_offsets: BTreeMap<String, u32>,
|
||||
fallback_scan_epoch: u32,
|
||||
exhausted_api_formats: BTreeSet<String>,
|
||||
@@ -427,11 +429,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
);
|
||||
|
||||
let ordering_config =
|
||||
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
state,
|
||||
routing_policy,
|
||||
)
|
||||
.await;
|
||||
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(routing_policy);
|
||||
|
||||
Self {
|
||||
state,
|
||||
@@ -460,6 +458,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
requested_name_offsets: BTreeMap::new(),
|
||||
scanned_rows_by_format: BTreeMap::new(),
|
||||
resolved_global_model_names: BTreeMap::new(),
|
||||
reserved_global_model_names: BTreeMap::new(),
|
||||
fallback_offsets: BTreeMap::new(),
|
||||
fallback_scan_epoch: 0,
|
||||
exhausted_api_formats: BTreeSet::new(),
|
||||
@@ -558,6 +557,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
self.requested_name_offsets.clear();
|
||||
self.scanned_rows_by_format.clear();
|
||||
self.resolved_global_model_names.clear();
|
||||
self.reserved_global_model_names.clear();
|
||||
self.fallback_offsets.clear();
|
||||
self.fallback_scan_epoch = self.fallback_scan_epoch.wrapping_add(1);
|
||||
self.exhausted_api_formats.clear();
|
||||
@@ -1188,6 +1188,34 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
|| self.exhausted_api_formats.contains(&normalized_api_format)
|
||||
}
|
||||
|
||||
/// Global model names are a reserved routing namespace, so a request that
|
||||
/// names one must not be answered by a provider whose own model merely
|
||||
/// carries that name as an upstream alias. Cached per routing model: the
|
||||
/// answer does not change between pages or API formats.
|
||||
async fn reserved_global_model_name(
|
||||
&mut self,
|
||||
rows: &[StoredMinimalCandidateSelectionRow],
|
||||
routing_model: &str,
|
||||
) -> Result<Option<String>, GatewayError> {
|
||||
if let Some(cached) = self.reserved_global_model_names.get(routing_model) {
|
||||
return Ok(cached.clone());
|
||||
}
|
||||
let state = self.state;
|
||||
let reserved_global_model_name =
|
||||
crate::data::candidate_selection::resolve_reserved_global_model_name(
|
||||
state.app().data.as_ref(),
|
||||
rows,
|
||||
routing_model,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
self.reserved_global_model_names.insert(
|
||||
routing_model.to_string(),
|
||||
reserved_global_model_name.clone(),
|
||||
);
|
||||
Ok(reserved_global_model_name)
|
||||
}
|
||||
|
||||
async fn build_page_outcome_from_rows(
|
||||
&mut self,
|
||||
candidate_api_format: &str,
|
||||
@@ -1219,15 +1247,17 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
if let Some(value) = self.resolved_global_model_names.get(normalized_api_format) {
|
||||
value.clone()
|
||||
} else {
|
||||
let Some(value) =
|
||||
resolve_requested_global_model_name_with_model_directives_and_request_operation(
|
||||
&rows,
|
||||
&routing_model,
|
||||
normalized_api_format,
|
||||
false,
|
||||
self.request_operation.as_deref(),
|
||||
)
|
||||
else {
|
||||
let reserved_global_model_name = self
|
||||
.reserved_global_model_name(&rows, &routing_model)
|
||||
.await?;
|
||||
let Some(value) = resolve_requested_global_model_name_with_reserved_global_model(
|
||||
&rows,
|
||||
&routing_model,
|
||||
normalized_api_format,
|
||||
false,
|
||||
self.request_operation.as_deref(),
|
||||
reserved_global_model_name.as_deref(),
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.resolved_global_model_names
|
||||
@@ -1293,9 +1323,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
.then_some(self.client_session_affinity.as_ref())
|
||||
.flatten(),
|
||||
self.ranking_seed,
|
||||
self.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
self.ordering_config,
|
||||
)
|
||||
.await?;
|
||||
let skipped_candidates = skipped_candidates
|
||||
@@ -1478,7 +1506,9 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::AppState;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::global_models::InMemoryGlobalModelReadRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::DataLayerError;
|
||||
use aether_data_contracts::repository::candidate_selection::{
|
||||
@@ -1486,6 +1516,9 @@ mod tests {
|
||||
StoredPoolKeyCandidateRowsByKeyIdsQuery, StoredPoolKeyCandidateRowsQuery,
|
||||
StoredProviderModelMapping, StoredRequestedModelCandidateRowsQuery,
|
||||
};
|
||||
use aether_data_contracts::repository::global_models::{
|
||||
GlobalModelReadRepository, StoredPublicGlobalModel,
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
@@ -1859,6 +1892,191 @@ mod tests {
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routing_policy_excludes_group_disabled_providers_from_candidate_pages() {
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||
standard_candidate_row("provider-disabled", "openai:chat", 0),
|
||||
standard_candidate_row("provider-enabled", "openai:chat", 1),
|
||||
]));
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||
);
|
||||
let auth_snapshot = unrestricted_auth_snapshot();
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let config = serde_json::from_value(serde_json::json!({
|
||||
"disabled_providers": ["provider-disabled"],
|
||||
"model_policies": [{
|
||||
"model": "*",
|
||||
"allowed_providers": ["provider-disabled", "provider-enabled"]
|
||||
}]
|
||||
}))
|
||||
.expect("routing config should parse");
|
||||
let routing_policy = aether_routing_core::resolve_routing_policy(
|
||||
&config,
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("routing-group-1"),
|
||||
group_version: Some(1),
|
||||
selection_source: "test",
|
||||
requested_model: "gpt-5",
|
||||
resolved_model: "gpt-5",
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &serde_json::json!({}),
|
||||
body: &serde_json::json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.expect("routing policy should resolve");
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&model_directive_policy,
|
||||
"openai:chat",
|
||||
"gpt-5",
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
Some(&routing_policy),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let page = cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("routing candidate scan should succeed")
|
||||
.expect("the enabled provider should remain");
|
||||
assert_eq!(
|
||||
page.candidates
|
||||
.iter()
|
||||
.map(|candidate| candidate.provider_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["provider-enabled"]
|
||||
);
|
||||
assert!(cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("routing scan should finish")
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_provider_enablement_filters_candidate_pages_without_affecting_other_models() {
|
||||
let mut rows = Vec::new();
|
||||
for model in ["model-a", "model-b", "model-c"] {
|
||||
for (provider, priority) in [
|
||||
("provider-legacy-disabled", 0),
|
||||
("provider-model-disabled", 1),
|
||||
("provider-other", 2),
|
||||
("provider-inactive", 3),
|
||||
] {
|
||||
let mut row = standard_candidate_row(provider, "openai:chat", priority);
|
||||
row.global_model_id = format!("global-{model}");
|
||||
row.global_model_name = model.into();
|
||||
row.model_provider_model_name = model.into();
|
||||
row.model_id = format!("{provider}-{model}");
|
||||
row.provider_is_active = provider != "provider-inactive";
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows));
|
||||
let app = AppState::new().unwrap().with_data_state_for_tests(
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||
);
|
||||
let auth = unrestricted_auth_snapshot();
|
||||
let directives = crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let config = serde_json::from_value(serde_json::json!({
|
||||
"disabled_providers": ["provider-legacy-disabled"],
|
||||
"model_policies": [
|
||||
{ "model": "model-a", "provider_enabled_overrides": {
|
||||
"provider-model-disabled": false, "provider-inactive": true
|
||||
} },
|
||||
{ "model": "model-b", "provider_enabled_overrides": {
|
||||
"provider-legacy-disabled": true, "provider-inactive": true
|
||||
} }
|
||||
],
|
||||
"rules": [{ "id": "legacy-allowlist", "actions": [{
|
||||
"type": "restrict_providers", "provider_ids": [
|
||||
"provider-legacy-disabled", "provider-model-disabled", "provider-other", "provider-inactive"
|
||||
]
|
||||
}] }]
|
||||
})).unwrap();
|
||||
// Revisit A after B to exercise candidate caches shared by the app.
|
||||
for (model, expected) in [
|
||||
("model-a", vec!["provider-other"]),
|
||||
(
|
||||
"model-b",
|
||||
vec![
|
||||
"provider-legacy-disabled",
|
||||
"provider-model-disabled",
|
||||
"provider-other",
|
||||
],
|
||||
),
|
||||
("model-c", vec!["provider-model-disabled", "provider-other"]),
|
||||
("model-a", vec!["provider-other"]),
|
||||
] {
|
||||
let policy = aether_routing_core::resolve_routing_policy(
|
||||
&config,
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("group-1"),
|
||||
group_version: Some(1),
|
||||
selection_source: "test",
|
||||
requested_model: model,
|
||||
resolved_model: model,
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &serde_json::json!({}),
|
||||
body: &serde_json::json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&directives,
|
||||
"openai:chat",
|
||||
model,
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth,
|
||||
Some(&policy),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let mut providers = Vec::new();
|
||||
while let Some(page) = cursor.next_page().await.unwrap() {
|
||||
providers.extend(
|
||||
page.candidates
|
||||
.into_iter()
|
||||
.map(|candidate| candidate.provider_id),
|
||||
);
|
||||
}
|
||||
providers.sort();
|
||||
assert_eq!(
|
||||
providers, expected,
|
||||
"provider enablement must remain isolated for {model}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routing_policy_collects_candidate_pages_before_final_ranking() {
|
||||
let rows = (0..300)
|
||||
@@ -1881,6 +2099,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
@@ -1890,6 +2110,7 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
@@ -1945,6 +2166,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-fallback".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
@@ -1954,6 +2177,7 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
@@ -2092,6 +2316,96 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn public_global_model(name: &str) -> StoredPublicGlobalModel {
|
||||
StoredPublicGlobalModel {
|
||||
id: format!("global-model-{name}"),
|
||||
name: name.to_string(),
|
||||
display_name: None,
|
||||
is_active: true,
|
||||
default_price_per_request: None,
|
||||
default_tiered_pricing: None,
|
||||
supported_capabilities: None,
|
||||
config: None,
|
||||
usage_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The cursor provider reaches its upstream under a name that belongs to another
|
||||
/// global model. A `claude:messages` client asking for `gemini-3.8-flash` has to
|
||||
/// land on the provider bound to that global model — format conversion and all —
|
||||
/// rather than on the one that only borrows the name on the way out, which is the
|
||||
/// one an API-format-ordered scan reaches first.
|
||||
#[tokio::test]
|
||||
async fn paged_preselection_keeps_a_global_model_name_from_a_provider_alias() {
|
||||
let mut aliasing = standard_candidate_row("ursor", "claude:messages", 1);
|
||||
aliasing.global_model_id = "global-model-gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.global_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.model_provider_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.model_provider_model_mappings = Some(vec![StoredProviderModelMapping {
|
||||
name: "gemini-3.8-flash".to_string(),
|
||||
priority: 1,
|
||||
api_formats: None,
|
||||
endpoint_ids: None,
|
||||
operations: None,
|
||||
}]);
|
||||
|
||||
let mut bound = standard_candidate_row("anti", "gemini:generate_content", 2);
|
||||
bound.global_model_id = "global-model-gemini-3.8-flash".to_string();
|
||||
bound.global_model_name = "gemini-3.8-flash".to_string();
|
||||
bound.model_provider_model_name = "gemini-3.8-flash".to_string();
|
||||
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||
aliasing, bound,
|
||||
]));
|
||||
let global_models: Arc<dyn GlobalModelReadRepository> =
|
||||
Arc::new(InMemoryGlobalModelReadRepository::seed([
|
||||
public_global_model("gemini-3.8-flash"),
|
||||
public_global_model("gemini-3.8-flash-cursor"),
|
||||
]));
|
||||
let data_state =
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository)
|
||||
.with_global_model_reader(global_models);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let auth_snapshot = unrestricted_auth_snapshot();
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&model_directive_policy,
|
||||
"claude:messages",
|
||||
"gemini-3.8-flash",
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let page = cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("preselection should succeed")
|
||||
.expect("the bound provider should still be reachable");
|
||||
|
||||
assert_eq!(page.candidates.len(), 1);
|
||||
assert_eq!(page.candidates[0].provider_name, "anti");
|
||||
assert_eq!(page.candidates[0].global_model_name, "gemini-3.8-flash");
|
||||
assert_eq!(
|
||||
page.candidates[0].endpoint_api_format,
|
||||
"gemini:generate_content"
|
||||
);
|
||||
}
|
||||
|
||||
fn standard_candidate_row(
|
||||
provider_id: &str,
|
||||
api_format: &str,
|
||||
@@ -2177,6 +2491,19 @@ mod tests {
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
row.provider_id.as_str(),
|
||||
row.key_id.as_str(),
|
||||
"plain-upstream-key",
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
row.key_id.clone(),
|
||||
row.provider_id.clone(),
|
||||
@@ -2188,7 +2515,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -2543,7 +2870,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key");
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
@@ -2663,7 +2990,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key")
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
// Legacy keys deliberately disagree with the routing policy: the
|
||||
// resolved policy must be the only source of scheduler ordering.
|
||||
.with_system_config_values_for_tests([
|
||||
@@ -2683,6 +3010,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-codex-first".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
@@ -2692,6 +3021,7 @@ mod tests {
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: true,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
|
||||
@@ -19,12 +19,12 @@ pub(crate) use crate::ai_serving::{
|
||||
GEMINI_FILES_UPLOAD_PLAN_KIND, GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND,
|
||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND,
|
||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
||||
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
|
||||
pub(crate) use aether_ai_serving::AiRequestedModelFamily as RequestedModelFamily;
|
||||
|
||||
@@ -6,12 +6,13 @@ use crate::ai_serving::planner::common::{
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND, GEMINI_FILES_LIST_PLAN_KIND,
|
||||
GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND, GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_EMBEDDING_SYNC_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
||||
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
use crate::ai_serving::planner::plan_builders::{
|
||||
build_gemini_stream_plan_from_decision, build_gemini_sync_plan_from_decision,
|
||||
@@ -101,7 +102,9 @@ fn build_sync_plan_payload_from_decision(
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND => {
|
||||
build_openai_responses_sync_plan_from_decision(parts, body_json, payload, false)?
|
||||
}
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND | OPENAI_SEARCH_SYNC_PLAN_KIND => {
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND
|
||||
| OPENAI_SEARCH_SYNC_PLAN_KIND
|
||||
| OPENAI_MEMORIES_SYNC_PLAN_KIND => {
|
||||
build_passthrough_sync_plan_from_decision(parts, payload)?
|
||||
}
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND => {
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||
};
|
||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
let Some(user_id) = decision
|
||||
.auth_context
|
||||
.as_ref()
|
||||
.filter(|auth_context| auth_context.access_allowed)
|
||||
.map(|auth_context| auth_context.user_id.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
|
||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
let Some(action) = state
|
||||
.video_tasks
|
||||
.prepare_openai_content_stream_action_for_user(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
user_id,
|
||||
)
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
|
||||
let auth_context = resolve_execution_runtime_auth_context(
|
||||
state,
|
||||
decision,
|
||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
)
|
||||
.await?;
|
||||
let Some(auth_context) = auth_context else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
&auth_context.user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||
plan_kind,
|
||||
parts.uri.path(),
|
||||
Some(body_json),
|
||||
Some(&auth_context),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
downstream_path = %parts.uri.path(),
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
client_api_format = %plan.client_api_format,
|
||||
upstream_base_url = ?upstream_base_url,
|
||||
upstream_url = %plan.url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
"gateway built local video follow-up sync decision payload"
|
||||
);
|
||||
|
||||
|
||||
@@ -13,11 +13,11 @@ use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
||||
use crate::ai_serving::transport::CodexFingerprintConvergenceContext;
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::ai_serving::{
|
||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||
CODEX_RESPONSES_LITE_HEADER,
|
||||
CODEX_RESPONSES_LITE_HEADER, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||
};
|
||||
use crate::cache::CacheLoadObserver;
|
||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||
"routing provider request violates provider contract";
|
||||
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||
"invalid provider request headers in routing mutation";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||
@@ -56,7 +60,7 @@ pub(crate) struct LocalRequestedModelDecisionInput {
|
||||
pub(crate) client_surface: Option<ClientSurface>,
|
||||
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
||||
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
||||
pub(crate) codex_fingerprint_context: Option<CodexFingerprintConvergenceContext>,
|
||||
pub(crate) provider_outbound_context: Option<ProviderOutboundRequestContext>,
|
||||
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
||||
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
||||
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
||||
@@ -119,6 +123,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||
websocket_continuation: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
let native_memories = decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND);
|
||||
let provider_api_format = decision
|
||||
.provider_api_format
|
||||
.clone()
|
||||
@@ -146,7 +151,12 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
)
|
||||
});
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
if native_memories {
|
||||
decision
|
||||
.provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
} else {
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
&mut decision.provider_request_headers,
|
||||
decision.provider_request_body.as_ref(),
|
||||
provider_type.as_str(),
|
||||
@@ -155,6 +165,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
model_capabilities.as_ref(),
|
||||
);
|
||||
}
|
||||
|
||||
let Some(context) = input.routing_context.as_ref() else {
|
||||
// Cache identity headers are projected only at the terminal boundary. Any non-empty
|
||||
@@ -168,7 +179,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -231,7 +242,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -256,7 +267,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_headers.insert(HeaderName::from_static(name), value);
|
||||
}
|
||||
}
|
||||
if original_provider_request_body.is_some() {
|
||||
if original_provider_request_body.is_some() && !native_memories {
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
@@ -312,10 +323,13 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
)
|
||||
}
|
||||
}
|
||||
.map_err(|violation| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
||||
})?;
|
||||
.map_err(|_| invalid_routing_provider_contract())?;
|
||||
}
|
||||
if native_memories {
|
||||
crate::ai_serving::transport::enforce_same_format_provider_api_operation_body_policy(
|
||||
&mut provider_request_body,
|
||||
Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize),
|
||||
);
|
||||
}
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
@@ -338,7 +352,11 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_type.as_str(),
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
if native_memories {
|
||||
provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
} else {
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
&mut provider_request_headers,
|
||||
Some(&provider_request_body),
|
||||
provider_type.as_str(),
|
||||
@@ -347,6 +365,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
model_capabilities.as_ref(),
|
||||
);
|
||||
}
|
||||
crate::ai_serving::apply_codex_openai_compact_terminal_headers(
|
||||
&mut provider_request_headers,
|
||||
provider_type.as_str(),
|
||||
@@ -357,7 +376,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
if original_provider_request_body.is_some() {
|
||||
decision.provider_request_body = Some(provider_request_body);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -367,7 +386,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_codex_fingerprint_convergence_to_decision(
|
||||
fn apply_provider_outbound_request_policies_to_decision(
|
||||
input: &LocalRequestedModelDecisionInput,
|
||||
decision: &mut AiExecutionDecision,
|
||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||
@@ -378,17 +397,26 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let Some(context) = input.codex_fingerprint_context.as_ref() else {
|
||||
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let applied = crate::ai_serving::transport::apply_codex_fingerprint_convergence_with_context(
|
||||
let native_context;
|
||||
let context = if decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND) {
|
||||
native_context = context
|
||||
.clone()
|
||||
.with_api_operation(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize);
|
||||
&native_context
|
||||
} else {
|
||||
context
|
||||
};
|
||||
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||
transport,
|
||||
provider_api_format,
|
||||
context,
|
||||
&mut decision.provider_request_headers,
|
||||
provider_request_body,
|
||||
);
|
||||
if applied {
|
||||
if results.iter().any(|result| result.was_applied()) {
|
||||
decision.prompt_cache_key = provider_request_body
|
||||
.get("prompt_cache_key")
|
||||
.and_then(Value::as_str)
|
||||
@@ -396,6 +424,31 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
}
|
||||
if results.is_empty() {
|
||||
return;
|
||||
}
|
||||
for result in &results {
|
||||
tracing::debug!(
|
||||
event_name = "provider_outbound_policy_evaluated",
|
||||
log_type = "event",
|
||||
policy = ?result.policy,
|
||||
outcome = ?result.outcome,
|
||||
reason = ?result.reason,
|
||||
mutation_scope = ?result.mutation_scope,
|
||||
identity_scope = ?result.identity_scope,
|
||||
"provider outbound request policy evaluated"
|
||||
);
|
||||
}
|
||||
let Some(serde_json::Value::Object(report_context)) = decision.report_context.as_mut() else {
|
||||
return;
|
||||
};
|
||||
report_context.insert(
|
||||
"provider_outbound_policies".to_string(),
|
||||
serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"results": results,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
||||
@@ -484,7 +537,7 @@ pub(crate) fn build_local_requested_model_decision_input(
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -499,27 +552,45 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
body_json: &Value,
|
||||
client_api_format: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
input.codex_fingerprint_context =
|
||||
input.provider_outbound_context =
|
||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||
let preferred_group = if explicit_group.is_none() && !input.auth_context.api_key_is_standalone {
|
||||
state
|
||||
.read_auth_api_key_feature_settings(
|
||||
&input.auth_context.user_id,
|
||||
&input.auth_context.api_key_id,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
.as_ref()
|
||||
.and_then(|settings| settings.get("routing_group_id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected_group = match state.routing_group_read_repository() {
|
||||
Some(repository) => {
|
||||
// Explicit non-default groups are authorized against principal
|
||||
// bindings, so both selection and its cache key must retain the
|
||||
// caller context. Only the implicit no-binding system-default
|
||||
// path is global and can skip the membership lookup.
|
||||
let principal_context_required = if explicit_group.is_some() {
|
||||
true
|
||||
} else {
|
||||
repository
|
||||
.has_any_routing_group_binding()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
||||
error.to_string(),
|
||||
))
|
||||
})?
|
||||
};
|
||||
let principal_context_required =
|
||||
if explicit_group.is_some() || preferred_group.is_some() {
|
||||
true
|
||||
} else {
|
||||
repository
|
||||
.has_any_routing_group_binding()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
||||
error.to_string(),
|
||||
))
|
||||
})?
|
||||
};
|
||||
let user_group_ids = if principal_context_required {
|
||||
let user_groups_lookup_started_at = std::time::Instant::now();
|
||||
let user_groups = state
|
||||
@@ -542,6 +613,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
principal_context_required.then(|| input.auth_context.api_key_id.clone());
|
||||
let selection_cache_key = routing_group_selection_cache_key(
|
||||
explicit_group.as_deref(),
|
||||
preferred_group.as_deref(),
|
||||
selection_user_id.as_deref(),
|
||||
selection_api_key_id.as_deref(),
|
||||
&user_group_ids,
|
||||
@@ -559,6 +631,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
repository.as_ref(),
|
||||
GatewayRoutingSelectionInput {
|
||||
explicit_group: explicit_group.as_deref(),
|
||||
preferred_group: preferred_group.as_deref(),
|
||||
user_id: selection_user_id.as_deref(),
|
||||
api_key_id: selection_api_key_id.as_deref(),
|
||||
user_group_ids: &user_group_ids,
|
||||
@@ -575,6 +648,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
|| {
|
||||
let repository = repository.clone();
|
||||
let explicit_group = explicit_group.clone();
|
||||
let preferred_group = preferred_group.clone();
|
||||
let user_id = selection_user_id.clone();
|
||||
let api_key_id = selection_api_key_id.clone();
|
||||
let user_group_ids = user_group_ids.clone();
|
||||
@@ -584,6 +658,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
repository.as_ref(),
|
||||
GatewayRoutingSelectionInput {
|
||||
explicit_group: explicit_group.as_deref(),
|
||||
preferred_group: preferred_group.as_deref(),
|
||||
user_id: user_id.as_deref(),
|
||||
api_key_id: api_key_id.as_deref(),
|
||||
user_group_ids: &user_group_ids,
|
||||
@@ -609,6 +684,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
selection.group.map(|group| {
|
||||
(
|
||||
Some(group.id),
|
||||
group.name,
|
||||
Some(group.version),
|
||||
group.config_json,
|
||||
selection.source,
|
||||
@@ -616,30 +692,28 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
})
|
||||
}
|
||||
None => {
|
||||
if explicit_group
|
||||
if let Some(requested_group) = explicit_group
|
||||
.or(preferred_group)
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
return Err(routing_selection_error(
|
||||
GatewayRoutingSelectionError::NotFound(explicit_group.unwrap_or_default()),
|
||||
GatewayRoutingSelectionError::NotFound(requested_group.to_string()),
|
||||
));
|
||||
}
|
||||
None
|
||||
return Err(routing_selection_error(
|
||||
GatewayRoutingSelectionError::NoDefault,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
let Some((group_id, group_version, group_config_json, selection_source)) = selected_group
|
||||
let Some((group_id, group_name, group_version, group_config_json, selection_source)) =
|
||||
selected_group
|
||||
else {
|
||||
input.client_session_affinity = client_session_affinity_from_api_request(
|
||||
client_api_format,
|
||||
&parts.headers,
|
||||
Some(body_json),
|
||||
);
|
||||
input.routing_policy = None;
|
||||
input.routing_trace_seed = None;
|
||||
input.routing_context = None;
|
||||
return Ok(());
|
||||
return Err(routing_selection_error(
|
||||
GatewayRoutingSelectionError::NoDefault,
|
||||
));
|
||||
};
|
||||
|
||||
if try_attach_static_default_routing_policy_to_input(
|
||||
@@ -652,6 +726,12 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
&group_config_json,
|
||||
selection_source.as_str(),
|
||||
)? {
|
||||
if let Some(policy) = input.routing_policy.as_mut() {
|
||||
policy.group_name = Some(group_name.clone());
|
||||
}
|
||||
if let Some(trace) = input.routing_trace_seed.as_mut() {
|
||||
trace.group_name = Some(group_name);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -737,6 +817,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
final_policy_resolve_started_at.elapsed().as_millis() as u64,
|
||||
);
|
||||
final_policy.mutation_plan = policy.mutation_plan.clone();
|
||||
final_policy.group_name = Some(group_name);
|
||||
input.routing_trace_seed = Some(build_routing_trace_seed(&final_policy, client_api_format));
|
||||
input.routing_policy = Some(final_policy);
|
||||
input.routing_context = Some(LocalRoutingRequestContext {
|
||||
@@ -863,10 +944,36 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
||||
GatewayRoutingSelectionError::Repository(message) => {
|
||||
GatewayError::Internal(format!("routing group repository lookup failed: {message}"))
|
||||
}
|
||||
error => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: error.to_string(),
|
||||
GatewayRoutingSelectionError::NoDefault => GatewayError::Client {
|
||||
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group was not found".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not enabled".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not allowed for this principal".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_contract() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_headers() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -891,6 +998,7 @@ fn routing_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<Stri
|
||||
|
||||
fn routing_group_selection_cache_key(
|
||||
explicit_group: Option<&str>,
|
||||
preferred_group: Option<&str>,
|
||||
user_id: Option<&str>,
|
||||
api_key_id: Option<&str>,
|
||||
user_group_ids: &[String],
|
||||
@@ -901,8 +1009,9 @@ fn routing_group_selection_cache_key(
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(
|
||||
"v1|explicit={}|user={}|api_key={}|groups={}",
|
||||
"v2|explicit={}|preferred={}|user={}|api_key={}|groups={}",
|
||||
escape_cache_key_part(explicit_group.unwrap_or_default()),
|
||||
escape_cache_key_part(preferred_group.unwrap_or_default()),
|
||||
escape_cache_key_part(user_id.unwrap_or_default()),
|
||||
escape_cache_key_part(api_key_id.unwrap_or_default()),
|
||||
groups
|
||||
@@ -921,14 +1030,9 @@ fn btree_headers_to_header_map(
|
||||
) -> Result<HeaderMap, GatewayError> {
|
||||
let mut output = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
||||
})?;
|
||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
||||
})?;
|
||||
let name = HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| invalid_routing_provider_headers())?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||
output.insert(name, value);
|
||||
}
|
||||
Ok(output)
|
||||
@@ -1105,10 +1209,13 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
use aether_data::repository::auth::{
|
||||
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
|
||||
use aether_data_contracts::repository::routing_profiles::{
|
||||
CreateRoutingGroupBindingRecord, CreateRoutingGroupRecord, RoutingGroupBindingSubject,
|
||||
RoutingGroupWriteRepository,
|
||||
RoutingGroupWriteRepository, UpdateRoutingGroupRecord,
|
||||
};
|
||||
use aether_provider_transport::snapshot::{
|
||||
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
|
||||
@@ -1119,12 +1226,14 @@ mod tests {
|
||||
fn explicit_routing_selection_cache_key_is_principal_specific() {
|
||||
let first = routing_group_selection_cache_key(
|
||||
Some("private"),
|
||||
None,
|
||||
Some("user-1"),
|
||||
Some("key-1"),
|
||||
&["team-1".to_string()],
|
||||
);
|
||||
let second = routing_group_selection_cache_key(
|
||||
Some("private"),
|
||||
None,
|
||||
Some("user-2"),
|
||||
Some("key-2"),
|
||||
&["team-2".to_string()],
|
||||
@@ -1151,6 +1260,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||
let secret = "private-group?token=Bearer-secret";
|
||||
|
||||
for error in [
|
||||
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||
] {
|
||||
let error = routing_selection_error(error);
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
ref message,
|
||||
} if !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||
let secret = "https://internal.example/?token=Bearer-secret";
|
||||
let contract_error = invalid_routing_provider_contract();
|
||||
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||
format!("Authorization: {secret}"),
|
||||
secret.to_string(),
|
||||
)]))
|
||||
.expect_err("invalid header should fail");
|
||||
|
||||
for (error, expected_message) in [
|
||||
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||
] {
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
ref message,
|
||||
} if message == expected_message && !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||
// AppState::new has no auth snapshot repository. Without the explicit
|
||||
@@ -1197,6 +1350,160 @@ mod tests {
|
||||
assert!(matches!(error, GatewayError::Internal(message) if message.contains("identity")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_routing_selection_applies_at_planner_and_invalidates_after_changes() {
|
||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(
|
||||
["api-key-1", "api-key-2"].map(|key_id| {
|
||||
(
|
||||
None,
|
||||
StoredAuthApiKeySnapshot::new(
|
||||
"user-1".into(),
|
||||
"alice".into(),
|
||||
None,
|
||||
"user".into(),
|
||||
"local".into(),
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
key_id.into(),
|
||||
Some(key_id.into()),
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
));
|
||||
let groups = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||
for (id, visible, is_default, multiplier) in [
|
||||
("default", false, true, 1.0),
|
||||
("discount", true, false, 0.5),
|
||||
("premium", true, false, 2.0),
|
||||
] {
|
||||
groups.create_routing_group(CreateRoutingGroupRecord {
|
||||
id: id.into(), name: format!("{id}-name"), description: None,
|
||||
enabled: true, is_system_default: is_default, sort_order: 0,
|
||||
config_json: json!({ "user_visible": visible, "billing_multiplier": multiplier }),
|
||||
version: 1, created_at: 1, updated_at: 1, published_at: None,
|
||||
}).await.unwrap();
|
||||
}
|
||||
let state = AppState::new().unwrap().with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_auth_api_key_repository_for_tests(auth_repository)
|
||||
.with_routing_group_repository_for_tests(groups.clone()),
|
||||
);
|
||||
for (key_id, group_id) in [("api-key-1", "discount"), ("api-key-2", "premium")] {
|
||||
assert!(state
|
||||
.set_user_api_key_feature_settings(
|
||||
"user-1",
|
||||
key_id,
|
||||
Some(json!({ "routing_group_id": group_id }))
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some());
|
||||
}
|
||||
let (parts, _) = http::Request::builder().body(()).unwrap().into_parts();
|
||||
let (header_parts, _) = http::Request::builder()
|
||||
.header(ROUTING_GROUP_HEADER, "premium")
|
||||
.body(())
|
||||
.unwrap()
|
||||
.into_parts();
|
||||
|
||||
async fn attach(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
key_id: &str,
|
||||
) -> Result<LocalRequestedModelDecisionInput, GatewayError> {
|
||||
let mut input = sample_decision_input();
|
||||
input.auth_context.api_key_id = key_id.into();
|
||||
input.auth_snapshot.api_key_id = key_id.into();
|
||||
attach_routing_policy_to_local_requested_model_input(
|
||||
state,
|
||||
parts,
|
||||
&mut input,
|
||||
&json!({ "model": "gpt-5" }),
|
||||
"openai:chat",
|
||||
)
|
||||
.await?;
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
// Revisit the first key after the second to exercise both cached choices.
|
||||
for (key_id, group_id, multiplier) in [
|
||||
("api-key-1", "discount", 0.5),
|
||||
("api-key-2", "premium", 2.0),
|
||||
("api-key-1", "discount", 0.5),
|
||||
] {
|
||||
let input = attach(&state, &parts, key_id).await.unwrap();
|
||||
let policy = input.routing_policy.as_ref().unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some(group_id));
|
||||
assert_eq!(policy.selection_source, "api_key_selection");
|
||||
assert_eq!(policy.billing_multiplier, multiplier);
|
||||
assert_eq!(
|
||||
input
|
||||
.routing_trace_seed
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.billing_multiplier,
|
||||
Some(multiplier)
|
||||
);
|
||||
}
|
||||
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||
let policy = header.routing_policy.unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some("premium"));
|
||||
assert_eq!(policy.selection_source, "explicit_header");
|
||||
|
||||
groups
|
||||
.update_routing_group(
|
||||
"discount",
|
||||
UpdateRoutingGroupRecord {
|
||||
config_json: Some(json!({ "user_visible": false, "billing_multiplier": 0.5 })),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
state.invalidate_provider_routing_caches();
|
||||
assert!(matches!(
|
||||
attach(&state, &parts, "api-key-1").await,
|
||||
Err(GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
..
|
||||
})
|
||||
));
|
||||
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||
assert_eq!(
|
||||
header.routing_policy.unwrap().group_id.as_deref(),
|
||||
Some("premium")
|
||||
);
|
||||
|
||||
assert!(state
|
||||
.set_user_api_key_feature_settings("user-1", "api-key-1", None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some());
|
||||
let cleared = attach(&state, &parts, "api-key-1").await.unwrap();
|
||||
let policy = cleared.routing_policy.unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some("default"));
|
||||
assert_eq!(policy.selection_source, "system_default");
|
||||
assert_eq!(policy.billing_multiplier, 1.0);
|
||||
// Clearing one key's preference must not disturb the other key's selection.
|
||||
let other = attach(&state, &parts, "api-key-2").await.unwrap();
|
||||
assert_eq!(
|
||||
other.routing_policy.unwrap().group_id.as_deref(),
|
||||
Some("premium")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_routing_attachment_authorizes_and_caches_per_principal() {
|
||||
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||
@@ -1207,7 +1514,8 @@ mod tests {
|
||||
description: None,
|
||||
enabled: true,
|
||||
is_system_default: false,
|
||||
config_json: json!({}),
|
||||
sort_order: 0,
|
||||
config_json: json!({"billing_multiplier": 0.5}),
|
||||
version: 1,
|
||||
created_at: 1,
|
||||
updated_at: 1,
|
||||
@@ -1253,6 +1561,11 @@ mod tests {
|
||||
.as_ref()
|
||||
.expect("explicit selection should attach routing policy");
|
||||
assert_eq!(policy.group_id.as_deref(), Some("private-group"));
|
||||
assert_eq!(policy.group_name.as_deref(), Some("private"));
|
||||
assert_eq!(policy.billing_multiplier, 0.5);
|
||||
let trace = allowed.routing_trace_seed.as_ref().unwrap();
|
||||
assert_eq!(trace.group_name.as_deref(), Some("private"));
|
||||
assert_eq!(trace.billing_multiplier, Some(0.5));
|
||||
assert_eq!(policy.selection_source, "explicit_header");
|
||||
|
||||
let mut denied = sample_decision_input();
|
||||
@@ -1327,7 +1640,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1576,7 +1889,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1620,7 +1933,7 @@ mod tests {
|
||||
assert_eq!(policy.group_version, Some(4));
|
||||
assert_eq!(
|
||||
policy.priority_mode,
|
||||
aether_routing_core::RoutingSetPriorityMode::GlobalKey
|
||||
aether_routing_core::RoutingSetPriorityMode::Provider
|
||||
);
|
||||
assert_eq!(
|
||||
policy.scheduling_mode,
|
||||
@@ -1646,7 +1959,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -1715,6 +2028,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_codex_provider_outbound_policies_are_terminal_noop() {
|
||||
let mut input = sample_decision_input();
|
||||
input.routing_context = None;
|
||||
input.provider_outbound_context = Some(ProviderOutboundRequestContext::new(
|
||||
"logical-turn",
|
||||
1_700_000_000_123,
|
||||
));
|
||||
let mut decision = sample_codex_fingerprint_decision();
|
||||
decision.provider_type = Some("openai".to_string());
|
||||
decision.provider_api_format = Some("openai:responses".to_string());
|
||||
decision.client_api_format = Some("openai:responses".to_string());
|
||||
let mut transport = sample_codex_fingerprint_transport();
|
||||
transport.provider.provider_type = "openai".to_string();
|
||||
|
||||
let original_headers = decision.provider_request_headers.clone();
|
||||
let original_body = decision.provider_request_body.clone();
|
||||
apply_provider_request_routing_policy_to_decision(&input, &mut decision, Some(&transport))
|
||||
.expect("non-Codex terminal finalization should succeed");
|
||||
|
||||
assert_eq!(decision.provider_request_headers, original_headers);
|
||||
assert_eq!(decision.provider_request_body, original_body);
|
||||
assert!(decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
||||
let transport = sample_codex_fingerprint_transport();
|
||||
@@ -1731,8 +2073,8 @@ mod tests {
|
||||
});
|
||||
let mut with_mutation = sample_decision_input();
|
||||
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
||||
input.codex_fingerprint_context = Some(
|
||||
CodexFingerprintConvergenceContext::new(
|
||||
input.provider_outbound_context = Some(
|
||||
ProviderOutboundRequestContext::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
1_756_668_000_000,
|
||||
)
|
||||
@@ -1801,6 +2143,25 @@ mod tests {
|
||||
);
|
||||
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
||||
|
||||
let policy_results = decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.and_then(|policies| policies.get("results"))
|
||||
.and_then(Value::as_array)
|
||||
.expect("provider policy results");
|
||||
assert_eq!(
|
||||
policy_results.len(),
|
||||
1,
|
||||
"policy result count at {exit_name}"
|
||||
);
|
||||
assert_eq!(
|
||||
policy_results[0]["policy"],
|
||||
json!("codex_fingerprint_convergence")
|
||||
);
|
||||
assert_eq!(policy_results[0]["outcome"], json!("applied"));
|
||||
assert_eq!(policy_results[0]["reason"], json!("applied"));
|
||||
|
||||
let header_metadata: Value =
|
||||
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
||||
.expect("header turn metadata");
|
||||
|
||||
@@ -7,7 +7,7 @@ use crate::ai_serving::transport::{
|
||||
build_gemini_cli_v1internal_request, build_standard_provider_request_headers,
|
||||
GatewayProviderTransportSnapshot, GeminiCliRequestAuth, GeminiCliRequestAuthSupport,
|
||||
GeminiCliRequestEnvelopeSupport, StandardProviderRequestHeaders,
|
||||
StandardProviderRequestHeadersInput, GEMINI_CLI_USER_AGENT,
|
||||
StandardProviderRequestHeadersInput,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
@@ -64,8 +64,10 @@ pub(crate) async fn build_gemini_cli_v1internal_provider_request(
|
||||
)
|
||||
.ok_or(GeminiCliV1InternalRequestError::UpstreamUrlUnavailable)?;
|
||||
|
||||
let extra_headers =
|
||||
BTreeMap::from([("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string())]);
|
||||
let extra_headers = BTreeMap::from([(
|
||||
"user-agent".to_string(),
|
||||
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||
)]);
|
||||
let headers = build_standard_provider_request_headers(StandardProviderRequestHeadersInput {
|
||||
transport: &payload.transport,
|
||||
provider_api_format: input.provider_api_format,
|
||||
|
||||
+6
-9
@@ -26,7 +26,6 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{
|
||||
@@ -141,10 +140,9 @@ pub(crate) async fn materialize_local_same_format_provider_candidate_attempts(
|
||||
current_unix_secs(),
|
||||
false,
|
||||
spec.operation.map(|operation| operation.as_str()),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let outcome = materialize_local_execution_candidates_with_serving(
|
||||
@@ -251,10 +249,9 @@ pub(crate) async fn build_local_same_format_provider_candidate_attempt_source<'a
|
||||
current_unix_secs(),
|
||||
false,
|
||||
spec.operation.map(|operation| operation.as_str()),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -21,10 +21,11 @@ use crate::ai_serving::transport::{
|
||||
build_same_format_provider_headers, resolve_local_gemini_cli_request_auth,
|
||||
GeminiCliRequestAuth, GeminiCliRequestAuthSupport, GeminiCliRequestEnvelopeSupport,
|
||||
GrokHeaderInput, SameFormatProviderCompatibilityEdit,
|
||||
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput,
|
||||
GEMINI_CLI_USER_AGENT, GROK_CHAT_PATH,
|
||||
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput, GROK_CHAT_PATH,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, CODEX_RESPONSES_LITE_HEADER,
|
||||
};
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
mod policy;
|
||||
@@ -255,7 +256,9 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
// re-enforce stream-field policy afterward.
|
||||
// Kiro behavior classification already hard-requires upstream streaming,
|
||||
// and the Kiro envelope does not use a top-level body stream field.
|
||||
if prepared.kiro_auth.is_none() {
|
||||
if prepared.kiro_auth.is_none()
|
||||
&& spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize)
|
||||
{
|
||||
enforce_provider_body_stream_policy(
|
||||
&mut base_provider_request_body,
|
||||
prepared.provider_api_format.as_str(),
|
||||
@@ -275,7 +278,8 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
prepared.mapped_model.as_str(),
|
||||
source_model,
|
||||
);
|
||||
if let Err(violation) =
|
||||
if spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||
if let Err(violation) =
|
||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
&mut base_provider_request_body,
|
||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||
@@ -313,6 +317,21 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Same-format requests skip `apply_transport_request_body_semantics`, so the opt-in
|
||||
// Claude Code body mimicry has to be applied here as well.
|
||||
if crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
prepared.provider_api_format.as_str(),
|
||||
) {
|
||||
compatibility_edits.push(SameFormatProviderCompatibilityEdit {
|
||||
field: "body".to_string(),
|
||||
action: SameFormatProviderCompatibilityEditAction::ProviderCompatibilityRewrite,
|
||||
detail: "applied Claude Code body mimicry for provider compatibility".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let antigravity_auth = if prepared.is_antigravity {
|
||||
let mut antigravity_support = classify_local_antigravity_request_support(
|
||||
@@ -513,7 +532,10 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
.map(build_antigravity_static_identity_headers)
|
||||
.unwrap_or_default();
|
||||
if prepared.behavior.is_gemini_cli {
|
||||
extra_headers.insert("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string());
|
||||
extra_headers.insert(
|
||||
"user-agent".to_string(),
|
||||
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||
);
|
||||
}
|
||||
let Some(mut provider_request_headers) = (if is_grok {
|
||||
build_grok_browser_headers(GrokHeaderInput {
|
||||
@@ -583,6 +605,16 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
source_model,
|
||||
codex_model_capabilities.as_ref(),
|
||||
);
|
||||
if spec.operation == Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||
provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
provider_request_headers.insert("accept".to_string(), "application/json".to_string());
|
||||
}
|
||||
crate::ai_serving::transport::xai::insert_cli_identity_headers_if_needed(
|
||||
transport.as_ref(),
|
||||
prepared.provider_api_format.as_str(),
|
||||
&mut provider_request_headers,
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
redaction.redacted,
|
||||
|
||||
@@ -26,6 +26,7 @@ use super::{
|
||||
LocalSameFormatProviderCandidateAttemptSource, LocalSameFormatProviderDecisionInput,
|
||||
LocalSameFormatProviderSpec,
|
||||
};
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
|
||||
pub(crate) struct LocalSameFormatProviderSyncAttemptSource<'a> {
|
||||
state: &'a AppState,
|
||||
@@ -189,6 +190,13 @@ pub(crate) async fn build_local_stream_attempt_source<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
@@ -234,6 +242,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncA
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt>
|
||||
for LocalSameFormatProviderStreamAttemptSource<'_>
|
||||
{
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_stream_attempt(attempt).await? {
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
||||
use super::super::LocalSameFormatProviderSpec;
|
||||
use crate::ai_serving::transport::{
|
||||
build_same_format_provider_request_body as build_same_format_provider_request_body_impl,
|
||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy as build_same_format_provider_request_body_with_compatibility_report_impl,
|
||||
build_same_format_provider_request_body_for_operation as build_same_format_provider_request_body_with_compatibility_report_impl,
|
||||
SameFormatProviderFamily, SameFormatProviderRequestBodyInput,
|
||||
SameFormatProviderRequestBodyOutput,
|
||||
};
|
||||
@@ -69,6 +69,7 @@ pub(crate) fn build_same_format_provider_request_body_with_compatibility_report(
|
||||
enable_model_directives,
|
||||
},
|
||||
reasoning_replay_policy,
|
||||
spec.operation,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,11 @@ use aether_ai_serving::{
|
||||
provider_stream_event_api_format_for_provider_type as ai_provider_stream_event_api_format_for_provider_type,
|
||||
AiExecutionReportContextParts, AiRequestOrigin, STICKY_KEY_ATTEMPTS_REPORT_FIELD,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
BillingMultiplierSnapshot, BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY,
|
||||
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY, ROUTING_GROUP_ID_METADATA_KEY,
|
||||
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||
};
|
||||
use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use aether_runtime_state::RuntimeLockLease;
|
||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
||||
@@ -21,7 +26,8 @@ use crate::client_session_affinity::{
|
||||
};
|
||||
use crate::orchestration::{
|
||||
insert_pool_key_lease_report_context_fields, ExecutionAttemptIdentity,
|
||||
ROUTING_POOL_POLICY_OVERRIDE_REPORT_FIELD, SCHEDULER_AFFINITY_EPOCH_REPORT_FIELD,
|
||||
ROUTING_EXECUTION_POLICY_REPORT_FIELD, ROUTING_POOL_POLICY_OVERRIDE_REPORT_FIELD,
|
||||
SCHEDULER_AFFINITY_EPOCH_REPORT_FIELD,
|
||||
};
|
||||
use crate::scheduler::affinity::insert_scheduler_affinity_policy_report_context_field;
|
||||
|
||||
@@ -75,15 +81,57 @@ pub(crate) fn build_local_execution_report_context(
|
||||
let RequestOrigin {
|
||||
client_ip,
|
||||
user_agent,
|
||||
forwarded_headers_trusted,
|
||||
} = parts
|
||||
.request_origin
|
||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
||||
let original_headers =
|
||||
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||
parts.original_request_body_json,
|
||||
parts.original_request_body_base64,
|
||||
);
|
||||
let mut extra_fields = parts.extra_fields;
|
||||
// Always overwrite caller-supplied extras with the planner's immutable policy snapshot.
|
||||
let billing_multiplier = parts
|
||||
.routing_policy
|
||||
.map(|policy| policy.billing_multiplier)
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.unwrap_or(1.0);
|
||||
extra_fields.insert(
|
||||
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY.to_string(),
|
||||
Value::from(billing_multiplier),
|
||||
);
|
||||
extra_fields.insert(
|
||||
BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY.to_string(),
|
||||
serde_json::to_value(
|
||||
BillingMultiplierSnapshot::from_factors(BTreeMap::from([(
|
||||
"routing_group".to_string(),
|
||||
billing_multiplier,
|
||||
)]))
|
||||
.expect("validated routing multiplier must produce a billing snapshot"),
|
||||
)
|
||||
.expect("validated billing snapshot must serialize"),
|
||||
);
|
||||
for (field, value) in [
|
||||
(
|
||||
ROUTING_GROUP_ID_METADATA_KEY,
|
||||
parts
|
||||
.routing_policy
|
||||
.and_then(|policy| policy.group_id.as_deref()),
|
||||
),
|
||||
(
|
||||
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||
parts
|
||||
.routing_policy
|
||||
.and_then(|policy| policy.group_name.as_deref()),
|
||||
),
|
||||
] {
|
||||
extra_fields.remove(field);
|
||||
if let Some(value) = value {
|
||||
extra_fields.insert(field.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
}
|
||||
if let Some(value) = parts
|
||||
.client_session_affinity
|
||||
.and_then(client_session_affinity_report_context_value)
|
||||
@@ -105,13 +153,20 @@ pub(crate) fn build_local_execution_report_context(
|
||||
value,
|
||||
);
|
||||
}
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
if forwarded_headers_trusted {
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
}
|
||||
}
|
||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||
if let Some(policy) = parts.routing_policy {
|
||||
if let Ok(value) = serde_json::to_value(&policy.execution_policy) {
|
||||
extra_fields.insert(ROUTING_EXECUTION_POLICY_REPORT_FIELD.to_string(), value);
|
||||
}
|
||||
}
|
||||
if let Some(override_policy) = parts
|
||||
.routing_policy
|
||||
.and_then(|policy| policy.pool_policy_overrides.get(parts.provider_id))
|
||||
@@ -183,6 +238,17 @@ pub(crate) fn build_local_execution_report_context(
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_report_context_original_headers(
|
||||
headers: &http::HeaderMap,
|
||||
forwarded_headers_trusted: bool,
|
||||
) -> BTreeMap<String, String> {
|
||||
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||
if !forwarded_headers_trusted {
|
||||
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||
}
|
||||
collected
|
||||
}
|
||||
|
||||
fn insert_request_path_fields(
|
||||
extra_fields: &mut Map<String, Value>,
|
||||
request_path: Option<&str>,
|
||||
@@ -252,8 +318,8 @@ mod tests {
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::{
|
||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
||||
LocalExecutionReportContextParts,
|
||||
build_local_execution_report_context, collect_report_context_original_headers,
|
||||
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||
};
|
||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||
use crate::ai_serving::RequestOrigin;
|
||||
@@ -283,6 +349,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||
|
||||
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||
assert_eq!(
|
||||
untrusted.get("user-agent").map(String::as_str),
|
||||
Some("test-client")
|
||||
);
|
||||
|
||||
let trusted = collect_report_context_original_headers(&headers, true);
|
||||
assert_eq!(
|
||||
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||
Some("spoofed-ja3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||
let auth_context = ExecutionRuntimeAuthContext {
|
||||
@@ -300,6 +386,27 @@ mod tests {
|
||||
Some("codex".to_string()),
|
||||
Some("account=account-1;session=session-1".to_string()),
|
||||
);
|
||||
let mut routing_policy = aether_routing_core::resolve_routing_policy(
|
||||
&aether_routing_core::RoutingGroupConfig {
|
||||
billing_multiplier: 0.25,
|
||||
..Default::default()
|
||||
},
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("group-1"),
|
||||
group_version: Some(7),
|
||||
selection_source: "system_default",
|
||||
requested_model: "gpt-5",
|
||||
resolved_model: "gpt-5",
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &json!({}),
|
||||
body: &json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.expect("routing policy should resolve");
|
||||
routing_policy.group_name = Some("请求时的分组".to_string());
|
||||
|
||||
let report_context =
|
||||
build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||
@@ -333,20 +440,40 @@ mod tests {
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: Some("203.0.113.8".to_string()),
|
||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||
forwarded_headers_trusted: false,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: Some(&client_session_affinity),
|
||||
routing_policy: None,
|
||||
routing_policy: Some(&routing_policy),
|
||||
scheduler_affinity_epoch: None,
|
||||
sticky_key_attempts: None,
|
||||
client_requested_stream: false,
|
||||
upstream_is_stream: false,
|
||||
has_envelope: false,
|
||||
needs_conversion: false,
|
||||
extra_fields: Map::new(),
|
||||
extra_fields: Map::from_iter([
|
||||
(
|
||||
"billing_multiplier_snapshot".to_string(),
|
||||
json!({
|
||||
"version": 1, "factors": {"routing_group": 99.0}, "multiplier": 99.0
|
||||
}),
|
||||
),
|
||||
("routing_group_billing_multiplier".to_string(), json!(99)),
|
||||
("routing_group_id".to_string(), json!("forged-group")),
|
||||
("routing_group_name".to_string(), json!("forged-name")),
|
||||
]),
|
||||
});
|
||||
|
||||
assert_eq!(report_context["routing_group_billing_multiplier"], 0.25);
|
||||
assert_eq!(
|
||||
report_context["billing_multiplier_snapshot"],
|
||||
json!({
|
||||
"version": 1, "factors": {"routing_group": 0.25}, "multiplier": 0.25
|
||||
})
|
||||
);
|
||||
assert_eq!(report_context["routing_group_id"], "group-1");
|
||||
assert_eq!(report_context["routing_group_name"], "请求时的分组");
|
||||
assert_eq!(
|
||||
report_context["client_ip"],
|
||||
Value::String("203.0.113.8".to_string())
|
||||
@@ -485,7 +612,11 @@ mod tests {
|
||||
original_headers: &original_headers,
|
||||
request_path: None,
|
||||
request_query_string: None,
|
||||
request_origin: None,
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
forwarded_headers_trusted: true,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: None,
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::ai_serving::{
|
||||
resolve_gemini_files_sync_spec as resolve_sync_spec, LocalGeminiFilesSpec,
|
||||
};
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
|
||||
use self::decision::maybe_build_local_gemini_files_decision_payload_for_candidate;
|
||||
use self::support::{
|
||||
@@ -174,6 +175,13 @@ pub(crate) async fn build_local_gemini_files_stream_attempt_source_for_kind<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
@@ -212,6 +220,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptS
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_stream_attempt(attempt).await? {
|
||||
|
||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
||||
GeminiFilesRequestBodyError,
|
||||
};
|
||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
use super::support::{
|
||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
if matches!(
|
||||
spec_metadata.decision_kind,
|
||||
GEMINI_FILES_GET_PLAN_KIND
|
||||
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||
{
|
||||
mark_skipped_local_gemini_files_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"gemini_file_mapping_mismatch",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(skip_reason) =
|
||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||
{
|
||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
file_name,
|
||||
})
|
||||
}
|
||||
|
||||
async fn candidate_matches_owned_gemini_file_mapping(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
input: &LocalGeminiFilesDecisionInput,
|
||||
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||
) -> bool {
|
||||
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||
return false;
|
||||
};
|
||||
let user_id = input.auth_context.user_id.trim();
|
||||
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(mapping)) = state
|
||||
.find_active_gemini_file_mapping_for_owner(
|
||||
file_name.as_str(),
|
||||
&attempt.eligible.transport.key.id,
|
||||
user_id,
|
||||
crate::clock::current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
|
||||
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||
if suffix.is_empty() || suffix.contains('/') {
|
||||
return None;
|
||||
}
|
||||
Some(format!("files/{suffix}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_gemini_file_name_from_path;
|
||||
|
||||
#[test]
|
||||
fn normalizes_supported_gemini_file_object_paths() {
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||
Some("files/abc-123".to_string())
|
||||
);
|
||||
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalGeminiFilesCandidateAttempt;
|
||||
@@ -109,10 +108,9 @@ pub(super) async fn materialize_local_gemini_files_candidate_attempts(
|
||||
Some(&input.auth_snapshot),
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let outcome = materialize_local_execution_candidates_with_serving(
|
||||
@@ -186,10 +184,9 @@ pub(super) async fn build_local_gemini_files_candidate_attempt_source<'a>(
|
||||
Some(&input.auth_snapshot),
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
Ok(build_local_execution_candidate_attempt_source_with_serving(
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::ai_serving::{
|
||||
resolve_local_image_sync_spec as resolve_sync_spec,
|
||||
};
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
|
||||
use self::decision::maybe_build_local_openai_image_decision_payload_for_candidate;
|
||||
use self::support::{
|
||||
@@ -252,6 +253,13 @@ pub(crate) async fn build_local_image_stream_attempt_source_for_kind<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
@@ -290,6 +298,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptS
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_stream_attempt(attempt).await? {
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::candidate_preparation::{
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::{
|
||||
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
||||
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
||||
@@ -16,8 +17,8 @@ use crate::ai_serving::transport::{
|
||||
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_special_headers, build_chatgpt_web_image_request_body,
|
||||
build_codex_openai_image_api_provider_request_body,
|
||||
apply_codex_openai_special_headers, apply_xai_upstream_payload_edits,
|
||||
build_chatgpt_web_image_request_body, build_codex_openai_image_api_provider_request_body,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
|
||||
default_model_for_openai_image_operation, normalize_openai_image_request,
|
||||
@@ -210,7 +211,7 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
upstream_is_stream,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body) = provider_request_body else {
|
||||
let Some(mut provider_request_body) = provider_request_body else {
|
||||
mark_skipped_local_openai_image_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
@@ -228,6 +229,11 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
.await;
|
||||
return None;
|
||||
};
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
);
|
||||
let Some(mut provider_request_headers) = (if is_grok {
|
||||
build_grok_browser_headers(GrokHeaderInput {
|
||||
transport,
|
||||
@@ -338,6 +344,25 @@ async fn resolve_local_openai_image_to_gemini_candidate_payload_parts(
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let provider_api_format = "gemini:generate_content";
|
||||
|
||||
// The gemini:generate_content URL hook rewrites an Antigravity endpoint to
|
||||
// /v1internal:, and this image path has no v1internal envelope to match it.
|
||||
// Skip the candidate instead of posting a bare Gemini body that upstream
|
||||
// would only reject.
|
||||
if is_antigravity_provider_transport(transport) {
|
||||
mark_skipped_local_openai_image_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
|
||||
@@ -27,7 +27,6 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
||||
|
||||
@@ -128,10 +127,9 @@ pub(super) async fn list_local_openai_image_candidate_attempts(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -206,10 +204,9 @@ pub(super) async fn build_local_openai_image_candidate_attempt_source<'a>(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
||||
LocalVideoCreateSpec,
|
||||
};
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
|
||||
use self::decision::maybe_build_local_video_create_decision_payload_for_candidate;
|
||||
use self::support::{
|
||||
@@ -104,6 +105,13 @@ pub(crate) async fn build_local_video_sync_attempt_source_for_kind<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::ai_serving::planner::{
|
||||
build_ai_execution_decision_response, resolve_transport_request_encoding_policy,
|
||||
AiExecutionDecisionResponseParts,
|
||||
};
|
||||
use crate::ai_serving::transport::xai::video::is_native_video_request;
|
||||
use crate::ai_serving::transport::{
|
||||
resolve_transport_execution_timeouts, resolve_transport_profile,
|
||||
};
|
||||
@@ -33,7 +34,7 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
let Some(resolved) = resolve_local_video_create_candidate_payload_parts(
|
||||
state, parts, body_json, trace_id, input, &attempt, spec,
|
||||
)
|
||||
.await
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -52,9 +53,32 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
.await;
|
||||
let transport_profile = resolve_transport_profile(&transport);
|
||||
let mut extra_fields = serde_json::Map::new();
|
||||
if is_native_video_request(&transport.provider.provider_type, parts.uri.path()) {
|
||||
extra_fields.insert(
|
||||
"video_client_protocol".to_string(),
|
||||
serde_json::json!("xai"),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(proxy_value) = build_request_trace_proxy_value(Some(&transport), proxy.as_ref()) {
|
||||
extra_fields.insert("proxy".to_string(), proxy_value);
|
||||
}
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai") {
|
||||
extra_fields.insert("video_provider_xai".into(), serde_json::json!(true));
|
||||
if let Some(duration) = resolved.provider_request_body.get("duration") {
|
||||
extra_fields.insert("video_duration".into(), duration.clone());
|
||||
}
|
||||
if parts.uri.path() == "/openai/v1/videos" {
|
||||
extra_fields.insert(
|
||||
"video_size".into(),
|
||||
body_json
|
||||
.get("size")
|
||||
.filter(|v| v.as_str().is_some_and(|s| !s.trim().is_empty()))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::json!("720x1280")),
|
||||
);
|
||||
}
|
||||
}
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||
auth_context: &input.auth_context,
|
||||
|
||||
@@ -3,15 +3,23 @@ use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::candidate_preparation::resolve_candidate_mapped_model;
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, resolve_candidate_mapped_model, OauthPreparationContext,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_video_create_spec_metadata;
|
||||
use crate::ai_serving::transport::xai::video::{
|
||||
convert_openai_video_request, is_explicit_native_video_path, is_native_video_request,
|
||||
};
|
||||
use crate::ai_serving::transport::{
|
||||
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
|
||||
resolve_video_create_auth, video_create_transport_unsupported_reason,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
};
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::AppState;
|
||||
use crate::ai_serving::{
|
||||
apply_xai_upstream_payload_edits, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
|
||||
PlannerAppState,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::support::{
|
||||
mark_skipped_local_video_candidate, mark_skipped_local_video_candidate_with_failure_diagnostic,
|
||||
@@ -37,11 +45,16 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
input: &LocalVideoCreateDecisionInput,
|
||||
attempt: &LocalVideoCreateCandidateAttempt,
|
||||
spec: LocalVideoCreateSpec,
|
||||
) -> Option<LocalVideoCreateCandidatePayloadParts> {
|
||||
) -> Result<Option<LocalVideoCreateCandidatePayloadParts>, GatewayError> {
|
||||
let spec_metadata = local_video_create_spec_metadata(spec);
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
if is_explicit_native_video_path(parts.uri.path())
|
||||
&& !transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let provider_family = provider_video_create_family(spec.family);
|
||||
let transport_unsupported_reason = video_create_transport_unsupported_reason(
|
||||
@@ -60,23 +73,39 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let auth = resolve_video_create_auth(transport, provider_family);
|
||||
let Some((auth_header, auth_value)) = auth else {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
PlannerAppState::new(state),
|
||||
transport,
|
||||
candidate,
|
||||
resolve_video_create_auth(transport, provider_family),
|
||||
OauthPreparationContext {
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_auth_unavailable",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
api_format: spec_metadata.api_format,
|
||||
operation: "video_create_candidate_request",
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(skip_reason) => {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let auth_header = prepared_candidate.auth_header;
|
||||
let auth_value = prepared_candidate.auth_value;
|
||||
|
||||
let mapped_model = match resolve_candidate_mapped_model(candidate) {
|
||||
Ok(mapped_model) => mapped_model,
|
||||
@@ -91,7 +120,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -117,10 +146,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(provider_request_body) = build_video_create_request_body(
|
||||
let Some(mut provider_request_body) = build_video_create_request_body(
|
||||
body_json,
|
||||
provider_family,
|
||||
&mapped_model,
|
||||
@@ -142,11 +171,28 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
&& !is_native_video_request(&transport.provider.provider_type, parts.uri.path())
|
||||
{
|
||||
provider_request_body =
|
||||
convert_openai_video_request(&provider_request_body).map_err(|message| {
|
||||
GatewayError::Client {
|
||||
status: http::StatusCode::BAD_REQUEST,
|
||||
message: message.to_string(),
|
||||
}
|
||||
})?;
|
||||
}
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
spec_metadata.api_format,
|
||||
);
|
||||
|
||||
let Some(provider_request_headers) =
|
||||
build_video_create_headers(ProviderVideoCreateHeadersInput {
|
||||
transport,
|
||||
headers: effective_headers,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
@@ -170,10 +216,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Some(LocalVideoCreateCandidatePayloadParts {
|
||||
Ok(Some(LocalVideoCreateCandidatePayloadParts {
|
||||
transport: Arc::clone(transport),
|
||||
auth_header,
|
||||
auth_value,
|
||||
@@ -181,7 +227,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
provider_request_headers,
|
||||
provider_request_body,
|
||||
upstream_url,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn provider_video_create_family(family: LocalVideoCreateFamily) -> ProviderVideoCreateFamily {
|
||||
|
||||
@@ -29,7 +29,6 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalVideoCreateCandidateAttempt;
|
||||
@@ -134,10 +133,9 @@ pub(super) async fn list_local_video_create_candidate_attempts(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -195,10 +193,9 @@ pub(super) async fn build_local_video_create_candidate_attempt_source<'a>(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||
input.routing_policy.as_ref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -504,10 +504,13 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
@@ -614,10 +617,13 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
||||
Some(&"acc-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
for name in ["x-client-request-id", "session-id", "thread-id"] {
|
||||
@@ -698,10 +704,13 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
||||
Some(&"kept-by-rule-request".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert_eq!(
|
||||
headers
|
||||
.keys()
|
||||
@@ -762,10 +771,13 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,39 +1,21 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub(crate) fn is_deepseek_provider(provider_type: &str, base_url: &str) -> bool {
|
||||
let provider_type = provider_type.trim().to_ascii_lowercase();
|
||||
if matches!(
|
||||
provider_type.as_str(),
|
||||
"deepseek" | "deepseek_openai" | "deepseek_anthropic" | "deepseek_compatible"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
pub(crate) fn is_deepseek_provider(_provider_type: &str, base_url: &str) -> bool {
|
||||
let Some(host) = base_url_host(base_url) else {
|
||||
return false;
|
||||
};
|
||||
host == "deepseek.com" || host.ends_with(".deepseek.com")
|
||||
}
|
||||
|
||||
fn is_deepseek_model(provider_model: &str) -> bool {
|
||||
let provider_model = provider_model.trim().to_ascii_lowercase();
|
||||
let leaf = provider_model
|
||||
.rsplit(['/', ':'])
|
||||
.next()
|
||||
.unwrap_or(provider_model.as_str());
|
||||
leaf == "deepseek" || leaf.starts_with("deepseek-") || leaf.starts_with("deepseek_")
|
||||
}
|
||||
|
||||
fn is_deepseek_upstream(provider_type: &str, base_url: &str, provider_model: &str) -> bool {
|
||||
is_deepseek_provider(provider_type, base_url) || is_deepseek_model(provider_model)
|
||||
// 仅官方接口启用专用兼容;供应商类型和模型名称不能代表第三方接口的行为。
|
||||
host == "api.deepseek.com"
|
||||
}
|
||||
|
||||
pub(crate) fn openai_responses_reasoning_replay_policy(
|
||||
provider_type: &str,
|
||||
base_url: &str,
|
||||
provider_model: &str,
|
||||
_provider_model: &str,
|
||||
) -> crate::ai_serving::OpenAiResponsesReasoningReplayPolicy {
|
||||
if is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if provider_type.trim().eq_ignore_ascii_case("xai") {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
} else if is_deepseek_provider(provider_type, base_url) {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
} else {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
@@ -47,11 +29,7 @@ pub(crate) fn apply_deepseek_tool_call_thinking_compat(
|
||||
provider_api_format: &str,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if !is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if !is_deepseek_provider(provider_type, base_url) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -155,6 +133,9 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
provider_request_body: &mut Value,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
// 携带 tools 时,所有历史 reasoning_content 都须完整回传,包括未调用工具的轮次。
|
||||
// 无 tools 时允许回传,且 prefix 续写需要保留输入;因此原样保留 messages,
|
||||
// 不删除思考内容,也不以空字符串冒充缺失内容,由上游校验请求是否完整。
|
||||
let disabled = source_disables_thinking(original_request_body, provider_request_body);
|
||||
set_deepseek_thinking_type(
|
||||
provider_request_body,
|
||||
@@ -170,33 +151,6 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
{
|
||||
object.remove("reasoning_effort");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(messages) = object.get_mut("messages").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
for message in messages {
|
||||
let Some(message_object) = message.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
let is_assistant = message_object
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"));
|
||||
if !is_assistant {
|
||||
continue;
|
||||
}
|
||||
if message_object
|
||||
.get("reasoning_content")
|
||||
.is_some_and(|value| !value.is_null())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
message_object.insert(
|
||||
"reasoning_content".to_string(),
|
||||
Value::String(String::new()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,8 +241,29 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_by_type_or_host() {
|
||||
assert!(is_deepseek_provider(
|
||||
fn xai_reasoning_policy_comes_from_provider_type() {
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"xai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"openai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_only_by_official_host() {
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://relay.example.com"
|
||||
));
|
||||
@@ -298,6 +273,16 @@ mod tests {
|
||||
));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com:443/v1"));
|
||||
assert!(!is_deepseek_provider("custom", "https://deepseek.com"));
|
||||
assert!(!is_deepseek_provider("custom", "deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider(
|
||||
"custom",
|
||||
" HTTPS://API.DEEPSEEK.COM:443/beta "
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://other.deepseek.com/v1"
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"custom",
|
||||
"https://example.com/deepseek"
|
||||
@@ -319,6 +304,15 @@ mod tests {
|
||||
"https://evil.example/[email protected]"
|
||||
));
|
||||
assert!(!is_deepseek_provider("custom", "ftp://api.deepseek.com/v1"));
|
||||
assert!(!is_deepseek_provider("deepseek", ""));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"deepseek",
|
||||
"https://deepseek.com/v1",
|
||||
"deepseek-chat",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"custom",
|
||||
@@ -341,7 +335,7 @@ mod tests {
|
||||
"https://api.b.ai/v1",
|
||||
"deepseek-v4-flash",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
@@ -353,6 +347,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn official_deepseek_host_enables_compat_without_type_or_model_hints() {
|
||||
for base_url in [
|
||||
"https://api.deepseek.com/v1",
|
||||
"https://api.deepseek.com/beta",
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "mapped-model",
|
||||
"messages": [{"role": "assistant", "content": "answer"}]
|
||||
});
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"custom",
|
||||
base_url,
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy("custom", base_url, "mapped-model"),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_deepseek_host_preserves_production_shaped_opaque_reasoning_replay() {
|
||||
let reasoning_items = (0..66)
|
||||
@@ -444,7 +465,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_adds_thinking_and_empty_reasoning_content() {
|
||||
fn openai_chat_deepseek_enables_thinking_without_fabricating_reasoning() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
@@ -467,11 +488,11 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
assert!(body["messages"][1].get("reasoning_content").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_relay_deepseek_model_adds_chat_thinking_compat() {
|
||||
fn custom_relay_deepseek_model_preserves_chat_request() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
@@ -484,6 +505,7 @@ mod tests {
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
||||
]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
@@ -493,8 +515,211 @@ mod tests {
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn third_party_hosts_ignore_deepseek_type_and_model_hints() {
|
||||
for provider_type in [
|
||||
"custom",
|
||||
"deepseek",
|
||||
"deepseek_openai",
|
||||
"deepseek_anthropic",
|
||||
"deepseek_compatible",
|
||||
] {
|
||||
for provider_model in [
|
||||
"other-model",
|
||||
"deepseek-chat",
|
||||
"deepseek-reasoner",
|
||||
"deepseek-v3",
|
||||
"deepseek-v4-flash",
|
||||
"vendor/deepseek-chat",
|
||||
"vendor:deepseek-reasoner",
|
||||
] {
|
||||
let base_url = "https://relay.example.com/v1";
|
||||
assert!(!is_deepseek_provider(provider_type, base_url));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
provider_type,
|
||||
base_url,
|
||||
provider_model
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
|
||||
for api_format in ["openai:chat", "claude:messages"] {
|
||||
let original = json!({
|
||||
"model": provider_model,
|
||||
"messages": [{
|
||||
"role": "assistant",
|
||||
"content": "answer",
|
||||
"reasoning_content": "original plan"
|
||||
}]
|
||||
});
|
||||
let mut body = original.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
provider_type,
|
||||
base_url,
|
||||
api_format,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
body, original,
|
||||
"{provider_type} / {provider_model} / {api_format}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_history_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Compare 9.11 and 9.8"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "9.8 is greater",
|
||||
"reasoning_content": "Compare the decimal places.\n9.80 > 9.11."
|
||||
},
|
||||
{"role": "user", "content": "Explain again"},
|
||||
{"role": "assistant", "content": "Compare 9.80 with 9.11"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_across_all_tool_turns() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}],
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"reasoning_content": "Check the weather before answering.\nKeep this full plan.",
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"}
|
||||
}]
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Cloudy"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "It is cloudy",
|
||||
"reasoning_content": "The weather result is available; summarize it."
|
||||
},
|
||||
{"role": "user", "content": "Should I take an umbrella?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "An umbrella may be useful",
|
||||
"reasoning_content": "Use the previous weather result without another tool call."
|
||||
},
|
||||
{"role": "user", "content": "Why?"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
let tools = body["tools"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["tools"], tools);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_does_not_fabricate_missing_tool_reasoning() {
|
||||
for tools in [
|
||||
json!([]),
|
||||
json!([{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}]),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": tools,
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "missing"},
|
||||
{"role": "assistant", "content": "null", "reasoning_content": null},
|
||||
{"role": "assistant", "content": "empty", "reasoning_content": ""},
|
||||
{"role": "assistant", "content": "answer", "reasoning_content": "original plan"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_prefix_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 1 + 1?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"prefix": true,
|
||||
"content": "",
|
||||
"reasoning_content": "Start by adding one to one."
|
||||
}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/beta",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::ai_serving::planner::spec_metadata::{
|
||||
};
|
||||
use crate::ai_serving::GatewayControlDecision;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
|
||||
use super::candidates::{
|
||||
build_local_standard_candidate_attempt_source, resolve_local_standard_decision_input,
|
||||
@@ -177,6 +178,13 @@ pub(crate) async fn build_local_stream_attempt_source<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
@@ -220,6 +228,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSour
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_stream_attempt(attempt).await? {
|
||||
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
|
||||
@@ -4,6 +4,10 @@ use std::sync::Arc;
|
||||
use aether_contracts::ResolvedTransportProfile;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::antigravity::{
|
||||
build_antigravity_v1internal_provider_request, AntigravityV1InternalRequestError,
|
||||
AntigravityV1InternalRequestInput, ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME,
|
||||
};
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
||||
OauthPreparationContext,
|
||||
@@ -26,6 +30,7 @@ use crate::ai_serving::planner::standard::{
|
||||
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
||||
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
||||
};
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::kiro::{
|
||||
build_kiro_provider_headers, build_kiro_provider_request_body,
|
||||
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
||||
@@ -587,7 +592,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -837,6 +842,29 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_antigravity_provider_transport(transport)
|
||||
{
|
||||
return Ok(build_antigravity_cross_format_payload_parts(
|
||||
state,
|
||||
parts,
|
||||
trace_id,
|
||||
body_json,
|
||||
input,
|
||||
attempt,
|
||||
transport,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
prepared_candidate.mapped_model,
|
||||
prepared_candidate.auth_header,
|
||||
prepared_candidate.auth_value,
|
||||
provider_request_body,
|
||||
upstream_is_stream,
|
||||
redaction.redacted,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_gemini_cli_provider_transport(transport)
|
||||
{
|
||||
@@ -963,6 +991,145 @@ fn apply_transport_request_body_semantics(
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_antigravity_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
trace_id: &str,
|
||||
original_body_json: &serde_json::Value,
|
||||
input: &LocalStandardDecisionInput,
|
||||
attempt: &LocalStandardCandidateAttempt,
|
||||
transport: &Arc<GatewayProviderTransportSnapshot>,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
mapped_model: String,
|
||||
auth_header: String,
|
||||
auth_value: String,
|
||||
gemini_request_body: Value,
|
||||
upstream_is_stream: bool,
|
||||
request_redacted: bool,
|
||||
) -> Option<LocalStandardCandidatePayloadParts> {
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let resolved =
|
||||
match build_antigravity_v1internal_provider_request(AntigravityV1InternalRequestInput {
|
||||
state,
|
||||
parts,
|
||||
transport,
|
||||
trace_id,
|
||||
mapped_model: &mapped_model,
|
||||
provider_api_format,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
request_headers: effective_headers,
|
||||
original_request_body: original_body_json,
|
||||
gemini_request_body: &gemini_request_body,
|
||||
upstream_is_stream,
|
||||
same_format: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(resolved) => resolved,
|
||||
Err(AntigravityV1InternalRequestError::TransportUnsupported) => {
|
||||
mark_skipped_local_standard_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::EnvelopeUnsupported) => {
|
||||
mark_skipped_local_standard_candidate_with_extra_data(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"provider_request_body_build_failed",
|
||||
request_body_build_failure_extra_data(
|
||||
original_body_json,
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::UpstreamUrlUnavailable) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"upstream_url_missing",
|
||||
CandidateFailureDiagnostic::upstream_url_missing(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_url",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::HeaderRulesApplyFailed) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_header_rules_apply_failed",
|
||||
CandidateFailureDiagnostic::header_rules_apply_failed(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_headers",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let mut provider_request_headers = resolved.headers.headers;
|
||||
apply_codex_openai_special_headers(
|
||||
&mut provider_request_headers,
|
||||
&resolved.body,
|
||||
effective_headers,
|
||||
resolved.transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
Some(trace_id),
|
||||
resolved.transport.key.decrypted_auth_config.as_deref(),
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
request_redacted,
|
||||
);
|
||||
|
||||
Some(LocalStandardCandidatePayloadParts {
|
||||
auth_header: resolved.headers.auth_header,
|
||||
auth_value: resolved.headers.auth_value,
|
||||
mapped_model,
|
||||
provider_api_format: provider_api_format.to_string(),
|
||||
provider_request_body: resolved.body,
|
||||
provider_request_headers,
|
||||
upstream_url: resolved.upstream_url,
|
||||
upstream_is_stream,
|
||||
envelope_name: Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME),
|
||||
transport: resolved.transport,
|
||||
transport_profile: None,
|
||||
request_redacted,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_gemini_cli_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::ai_serving::transport::apply_standard_provider_request_body_rules_wit
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_responses_chat_body_edits,
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives as surface_build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context as surface_build_cross_format_openai_chat_request_body,
|
||||
build_local_openai_chat_request_body_with_model_directives as surface_build_local_openai_chat_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -73,9 +73,11 @@ pub(crate) fn build_cross_format_openai_chat_request_body(
|
||||
let provider_request_body = surface_build_cross_format_openai_chat_request_body(
|
||||
body_json,
|
||||
mapped_model,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
user_api_key_id,
|
||||
)?;
|
||||
let mut provider_request_body =
|
||||
apply_standard_provider_request_body_rules_with_request_headers(
|
||||
@@ -125,3 +127,39 @@ pub(crate) fn build_cross_format_openai_chat_upstream_url(
|
||||
parts.uri.query(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_chat_route_preserves_tool_schema_and_alternate_responses_shape() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let body = json!({"model": "client", "messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{"type": "function", "function": {"name": "probe", "parameters": schema}}]});
|
||||
let responses_body = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for input in [body, responses_body] {
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output = build_cross_format_openai_chat_request_body(
|
||||
&input,
|
||||
"claude-test",
|
||||
provider,
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
||||
use crate::ai_serving::transport::apply_standard_provider_request_body_rules_with_request_headers;
|
||||
use crate::ai_serving::{
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope as surface_build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context as surface_build_cross_format_openai_responses_request_body,
|
||||
build_local_openai_responses_request_body_with_model_directives as surface_build_local_openai_responses_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -218,6 +218,7 @@ pub(crate) fn build_cross_format_openai_responses_request_body_with_codex_model_
|
||||
body_json,
|
||||
mapped_model,
|
||||
client_api_format,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
@@ -291,3 +292,38 @@ pub(crate) fn build_cross_format_openai_responses_upstream_url(
|
||||
parts.uri.query(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_responses_route_preserves_tool_schema_without_changing_public_gemini() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let input = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output =
|
||||
build_cross_format_openai_responses_request_body_with_codex_model_capabilities(
|
||||
&input,
|
||||
"claude-test",
|
||||
"openai:responses",
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
provider,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
Some("antigravity-schema-test"),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+16
-6
@@ -140,7 +140,7 @@ fn finalize_openai_chat_provider_request_body(
|
||||
mapped_model,
|
||||
source_model,
|
||||
);
|
||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
let finalization_failure = crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
provider_request_body,
|
||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||
source_api_format: "openai:chat",
|
||||
@@ -170,7 +170,17 @@ fn finalize_openai_chat_provider_request_body(
|
||||
provider_api_format,
|
||||
"openai_chat_request_finalization",
|
||||
)
|
||||
})
|
||||
});
|
||||
if finalization_failure.is_none() {
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Chat -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
provider_request_body,
|
||||
transport,
|
||||
provider_api_format,
|
||||
);
|
||||
}
|
||||
finalization_failure
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2183,7 +2193,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -2741,7 +2751,7 @@ mod tests {
|
||||
.provider_request_headers
|
||||
.get("x-client-version")
|
||||
.map(String::as_str),
|
||||
Some("1.2.3")
|
||||
Some("4.3.0")
|
||||
);
|
||||
assert_eq!(
|
||||
payload
|
||||
@@ -2761,9 +2771,9 @@ mod tests {
|
||||
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
||||
assert_eq!(
|
||||
payload.provider_request_body["userAgent"],
|
||||
"antigravity/cli/1.0.16 (aidev_client; os_type=linux; arch=arm64; auth_method=consumer)"
|
||||
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||
);
|
||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
||||
assert!(payload.provider_request_body.get("requestType").is_none());
|
||||
assert!(payload.provider_request_body.get("contents").is_none());
|
||||
assert!(payload.provider_request_body["request"]
|
||||
.get("contents")
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode};
|
||||
use async_trait::async_trait;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
@@ -119,6 +120,13 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
let select_started_at = std::time::Instant::now();
|
||||
let selected = self.next_execution_attempt_with_target_select().await?;
|
||||
@@ -199,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
|
||||
async fn next_raw_attempt_with_target_select(
|
||||
&mut self,
|
||||
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
||||
let select_window = openai_chat_stream_target_select_window();
|
||||
let select_window = openai_chat_stream_target_select_window_for_mode(
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.scheduling_mode),
|
||||
);
|
||||
if select_window <= 1 {
|
||||
return self.next_raw_attempt_linear().await;
|
||||
}
|
||||
@@ -357,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize {
|
||||
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
||||
}
|
||||
|
||||
fn openai_chat_stream_target_select_window_for_mode(
|
||||
scheduling_mode: Option<RoutingSchedulingMode>,
|
||||
) -> usize {
|
||||
if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) {
|
||||
return 1;
|
||||
}
|
||||
openai_chat_stream_target_select_window()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TargetSelectCandidateIdentity<'a> {
|
||||
provider_id: &'a str,
|
||||
@@ -566,4 +588,14 @@ mod tests {
|
||||
|
||||
assert_eq!(select_target_index(19, &choices), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_order_disables_stream_target_selection() {
|
||||
assert_eq!(
|
||||
openai_chat_stream_target_select_window_for_mode(Some(
|
||||
RoutingSchedulingMode::FixedOrder,
|
||||
)),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
use async_trait::async_trait;
|
||||
use tracing::warn;
|
||||
|
||||
@@ -92,6 +93,13 @@ pub(crate) async fn build_local_openai_chat_sync_attempt_source<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
|
||||
+12
-27
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiStreamAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_stream_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = effective_upstream_is_stream,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiSyncAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_sync_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = payload.upstream_is_stream,
|
||||
|
||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
||||
|
||||
use crate::ai_serving::build_request_trace_proxy_value;
|
||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::planner::report_context::{
|
||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
&resolved.transport,
|
||||
);
|
||||
|
||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
||||
debug!(
|
||||
event_name = "local_openai_responses_decision_payload_built",
|
||||
log_type = "debug",
|
||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
client_api_format = spec_metadata.api_format,
|
||||
provider_api_format = %resolved.provider_api_format,
|
||||
request_path = %parts.uri.path(),
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||
request_query = ?log_request_query,
|
||||
upstream_base_url = %log_base_url,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream = resolved.upstream_is_stream,
|
||||
has_envelope = resolved.envelope_name.is_some(),
|
||||
"gateway built local openai responses decision payload"
|
||||
|
||||
+14
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
||||
};
|
||||
use crate::ai_serving::planner::redaction::{
|
||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||
sanitize_upstream_url_for_log,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||
use crate::ai_serving::planner::standard::{
|
||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -636,6 +635,13 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
{
|
||||
log_responses_to_chat_tool_conversion(trace_id, body_json, &base_provider_request_body);
|
||||
}
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Responses -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
provider_api_format,
|
||||
);
|
||||
let provider_request_body = base_provider_request_body;
|
||||
|
||||
if let Some(kiro_auth) = kiro_auth.as_ref() {
|
||||
@@ -867,17 +873,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
||||
let log_custom_path = transport
|
||||
.endpoint
|
||||
.custom_path
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_upstream_url_resolved",
|
||||
@@ -893,12 +892,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
base_url = %log_base_url,
|
||||
custom_path = ?log_custom_path,
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
request_path = %parts.uri.path(),
|
||||
request_query = ?log_request_query,
|
||||
mapped_model = %mapped_model,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses upstream url"
|
||||
);
|
||||
@@ -2011,8 +2012,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
};
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||
log_type = "debug",
|
||||
@@ -2027,7 +2026,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses kiro upstream url"
|
||||
);
|
||||
|
||||
@@ -488,6 +488,7 @@ impl ResponsesWebSocketBodyNormalization {
|
||||
digest.update([match self.reasoning_replay_policy {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds => 0,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque => 1,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted => 2,
|
||||
}]);
|
||||
update_normalization_optional_json_digest(&mut digest, self.model_directive_patch.as_ref());
|
||||
digest.finalize().into()
|
||||
@@ -729,117 +730,6 @@ fn update_normalization_codex_capabilities_digest(
|
||||
update_normalization_string_vec_digest(digest, &capabilities.supported_service_tiers);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod continuation_fingerprint_tests {
|
||||
use http::HeaderValue;
|
||||
use serde_json::json;
|
||||
|
||||
use super::ResponsesWebSocketBodyNormalization;
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
||||
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
||||
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_changes_with_effective_contract() {
|
||||
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
||||
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
||||
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
||||
);
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_policy.continuation_fingerprint()
|
||||
);
|
||||
|
||||
let changed_patch = base
|
||||
.clone()
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_patch.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "x-contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
||||
second
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
||||
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "X-Contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules.clone());
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("disabled"));
|
||||
|
||||
assert_ne!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"a header that controls an effective body-rule condition remains part of the contract"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds one upstream decision for a Responses WebSocket turn. The session
|
||||
/// reuses this decision for same-model turns and invokes the planner again when
|
||||
/// a later `response.create` changes the public model.
|
||||
@@ -1058,3 +948,114 @@ async fn release_responses_websocket_planning_lease(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod continuation_fingerprint_tests {
|
||||
use http::HeaderValue;
|
||||
use serde_json::json;
|
||||
|
||||
use super::ResponsesWebSocketBodyNormalization;
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_is_stable_for_json_object_key_order() {
|
||||
let first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "high"}, "x": 1}));
|
||||
let second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_model_directive_patch_for_tests(json!({"x": 1, "reasoning": {"effort": "high"}}));
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_changes_with_effective_contract() {
|
||||
let base = ResponsesWebSocketBodyNormalization::for_tests("provider-model");
|
||||
let changed_policy = base.clone().with_reasoning_replay_policy_for_tests(
|
||||
OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque,
|
||||
);
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_policy.continuation_fingerprint()
|
||||
);
|
||||
|
||||
let changed_patch = base
|
||||
.clone()
|
||||
.with_model_directive_patch_for_tests(json!({"reasoning": {"effort": "low"}}));
|
||||
assert_ne!(
|
||||
base.continuation_fingerprint(),
|
||||
changed_patch.continuation_fingerprint()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_ignores_unrelated_volatile_request_headers() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "x-contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-1"));
|
||||
first
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-1"));
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-request-id", HeaderValue::from_static("request-2"));
|
||||
second
|
||||
.request_headers
|
||||
.insert("cf-ray", HeaderValue::from_static("edge-2"));
|
||||
|
||||
assert_eq!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"headers that no body-rule condition reads must not invalidate a persisted continuation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_fingerprint_tracks_headers_used_by_body_rule_conditions() {
|
||||
let body_rules = json!([{
|
||||
"action": "set",
|
||||
"path": "store",
|
||||
"value": false,
|
||||
"condition": {
|
||||
"source": "request_headers",
|
||||
"path": "X-Contract",
|
||||
"op": "eq",
|
||||
"value": "enabled"
|
||||
}
|
||||
}]);
|
||||
let mut first = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules.clone());
|
||||
first
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("enabled"));
|
||||
let mut second = ResponsesWebSocketBodyNormalization::for_tests("provider-model")
|
||||
.with_body_rules_for_tests(body_rules);
|
||||
second
|
||||
.request_headers
|
||||
.insert("x-contract", HeaderValue::from_static("disabled"));
|
||||
|
||||
assert_ne!(
|
||||
first.continuation_fingerprint(),
|
||||
second.continuation_fingerprint(),
|
||||
"a header that controls an effective body-rule condition remains part of the contract"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
use async_trait::async_trait;
|
||||
use tracing::warn;
|
||||
|
||||
@@ -161,6 +162,13 @@ pub(super) async fn build_local_stream_attempt_source<'a>(
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_sync_attempt(attempt).await? {
|
||||
@@ -204,6 +212,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAtte
|
||||
|
||||
#[async_trait]
|
||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStreamAttemptSource<'_> {
|
||||
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.execution_policy.clone())
|
||||
}
|
||||
|
||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||
match self.build_stream_attempt(attempt).await? {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::constants::{
|
||||
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
|
||||
};
|
||||
@@ -12,9 +10,8 @@ use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::GatewayError;
|
||||
|
||||
impl<'a> PlannerAppState<'a> {
|
||||
/// `ordering_config` is the request's routing-policy derived scheduler
|
||||
/// config (see `SchedulerOrderingConfig::from_routing_policy`). `None`
|
||||
/// falls back to the runtime default.
|
||||
/// `ordering_config` is the immutable scheduler snapshot derived from the
|
||||
/// request's resolved routing policy.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_candidates(
|
||||
self,
|
||||
@@ -26,7 +23,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
crate::scheduler::candidate::list_selectable_candidates(
|
||||
self.app().data.as_ref(),
|
||||
@@ -55,7 +52,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -90,7 +87,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
request_operation: Option<&str>,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -98,11 +95,13 @@ impl<'a> PlannerAppState<'a> {
|
||||
),
|
||||
GatewayError,
|
||||
> {
|
||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||
let wait_deadline = Instant::now() + wait_timeout;
|
||||
let mut attempt_now_unix_secs = now_unix_secs;
|
||||
loop {
|
||||
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||
self.app(),
|
||||
auth_snapshot,
|
||||
now_unix_secs,
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||
|attempt_now_unix_secs| async move {
|
||||
let result = crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons_for_request_operation(
|
||||
self.app().data.as_ref(),
|
||||
self.app(),
|
||||
@@ -119,21 +118,13 @@ impl<'a> PlannerAppState<'a> {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
||||
let auth_limit_blocked = crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
|
||||
&result.0, &result.1,
|
||||
) {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= wait_deadline {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let remaining = wait_deadline.duration_since(now);
|
||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
||||
attempt_now_unix_secs = current_unix_secs();
|
||||
}
|
||||
);
|
||||
Ok((result, auth_limit_blocked))
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -146,7 +137,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -177,15 +168,16 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
ordering_config: SchedulerOrderingConfig,
|
||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||
let wait_deadline = Instant::now() + wait_timeout;
|
||||
let mut attempt_now_unix_secs = now_unix_secs;
|
||||
|
||||
loop {
|
||||
let (result, auth_limit_blocked) = crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
||||
crate::scheduler::candidate::select_with_auth_concurrency_wait(
|
||||
self.app(),
|
||||
auth_snapshot,
|
||||
now_unix_secs,
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS),
|
||||
Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS),
|
||||
|attempt_now_unix_secs| {
|
||||
crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
|
||||
self.app().data.as_ref(),
|
||||
self.app(),
|
||||
candidate_api_format,
|
||||
@@ -196,20 +188,8 @@ impl<'a> PlannerAppState<'a> {
|
||||
attempt_now_unix_secs,
|
||||
ordering_config,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !auth_limit_blocked {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now >= wait_deadline {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
let remaining = wait_deadline.duration_since(now);
|
||||
tokio::time::sleep(wait_interval.min(remaining)).await;
|
||||
attempt_now_unix_secs = current_unix_secs();
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,13 +12,16 @@ pub(crate) use aether_ai_formats::api::{
|
||||
apply_codex_openai_responses_websocket_continuation_body_edits_with_source_model_and_capabilities,
|
||||
apply_codex_openai_special_headers, apply_model_directive_mapping_patch,
|
||||
apply_model_directive_overrides_from_model, apply_model_directive_overrides_from_request,
|
||||
apply_openai_responses_compact_special_body_edits, build_chatgpt_web_image_request_body,
|
||||
apply_openai_responses_compact_special_body_edits, apply_xai_upstream_payload_edits,
|
||||
apply_xai_upstream_payload_edits_with_client, build_chatgpt_web_image_request_body,
|
||||
build_codex_model_catalog_metadata, build_codex_openai_image_api_provider_request_body,
|
||||
build_core_error_body_for_client_format, build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context,
|
||||
build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_gemini_image_response_from_openai_image_response,
|
||||
build_gemini_image_response_from_openai_responses_image_response, build_generated_tool_call_id,
|
||||
@@ -160,7 +163,7 @@ pub(crate) use aether_ai_formats::api::{
|
||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
@@ -177,10 +180,11 @@ pub(crate) use aether_ai_formats::{
|
||||
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
||||
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
||||
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
||||
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
|
||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||
strip_incompatible_openai_responses_reasoning_items,
|
||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||
CODEX_CLIENT_VERSION, OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
};
|
||||
|
||||
pub(crate) fn plan_kind_matches_api_operation(
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||
};
|
||||
use aether_runtime_state::RuntimeState;
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::GatewayError;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||
|
||||
pub(crate) async fn hydrate_openai_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
request: &Value,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||
let runtime_state = state.runtime_state();
|
||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||
warn!(
|
||||
event_name = "openai_response_history_read_failed",
|
||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
let Some(payload) = payload else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&payload,
|
||||
) else {
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
event_name = "openai_response_history_decryption_failed",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway rejected undecryptable shared OpenAI response history"
|
||||
);
|
||||
return Err(GatewayError::Internal(
|
||||
"OpenAI response history decryption failed".to_string(),
|
||||
));
|
||||
};
|
||||
if let Err(error) =
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||
{
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_response_history_record(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
record: ResponseHistoryRecord,
|
||||
) {
|
||||
let runtime_state = state.runtime_state();
|
||||
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&record.payload,
|
||||
) else {
|
||||
warn!(
|
||||
event_name = "openai_response_history_encryption_unavailable",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway refused to persist unencrypted OpenAI response history"
|
||||
);
|
||||
return;
|
||||
};
|
||||
if let Err(error) = runtime_state
|
||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
||||
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_converted_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
report_context: &Value,
|
||||
response: Option<&Value>,
|
||||
) {
|
||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
||||
return;
|
||||
};
|
||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||
persist_response_history_record(runtime_state, record).await;
|
||||
persist_response_history_record(state, record).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||
};
|
||||
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||
|
||||
fn response_history_test_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("test state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||
MemoryRuntimeStateConfig::default(),
|
||||
)))
|
||||
}
|
||||
|
||||
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||
let expires_at_unix_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(3600);
|
||||
json!({
|
||||
"version": 1,
|
||||
"response_id": response_id,
|
||||
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||
"expires_at_unix_secs": expires_at_unix_secs,
|
||||
"transcript": [{"type": "message", "content": marker}],
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_encrypted_history_v1";
|
||||
let scope = "response-history-encrypted-scope";
|
||||
let marker = "private-response-history-marker";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, marker);
|
||||
|
||||
persist_response_history_record(
|
||||
&state,
|
||||
ResponseHistoryRecord {
|
||||
storage_key: storage_key.clone(),
|
||||
payload,
|
||||
ttl: Duration::from_secs(6 * 60 * 60),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stored = state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.expect("history should be persisted");
|
||||
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||
&stored
|
||||
));
|
||||
assert!(!stored.contains(marker));
|
||||
|
||||
hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
.expect("encrypted history should hydrate");
|
||||
assert!(crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_legacy_history_v1";
|
||||
let scope = "response-history-legacy-scope";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||
state
|
||||
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||
.await
|
||||
.expect("legacy history should store");
|
||||
|
||||
let result = hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
assert!(state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,11 +58,16 @@ pub(crate) mod windsurf {
|
||||
pub(crate) use aether_provider_transport::windsurf::*;
|
||||
}
|
||||
|
||||
pub(crate) mod xai {
|
||||
pub(crate) use aether_provider_transport::xai::*;
|
||||
}
|
||||
|
||||
pub(crate) use aether_provider_transport::{
|
||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_standard_provider_request_body_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_provider_outbound_request_policies,
|
||||
apply_standard_provider_request_body_rules,
|
||||
apply_standard_provider_request_body_rules_with_request_headers,
|
||||
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
||||
body_rules_handle_path, body_rules_have_enabled_rules,
|
||||
@@ -73,7 +78,7 @@ pub(crate) use aether_provider_transport::{
|
||||
build_local_openai_chat_upstream_url, build_local_openai_responses_upstream_url,
|
||||
build_openai_image_headers, build_openai_image_upstream_url, build_passthrough_headers,
|
||||
build_request_trace_proxy_value, build_same_format_provider_headers,
|
||||
build_same_format_provider_request_body,
|
||||
build_same_format_provider_request_body, build_same_format_provider_request_body_for_operation,
|
||||
build_same_format_provider_request_body_with_compatibility_report,
|
||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy,
|
||||
build_same_format_provider_upstream_url, build_standard_plan_fallback_headers,
|
||||
@@ -112,7 +117,11 @@ pub(crate) use aether_provider_transport::{
|
||||
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
||||
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
||||
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
||||
ProviderOpenAiImageHeadersInput, ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
ProviderOpenAiImageHeadersInput, ProviderOutboundRequestContext,
|
||||
ProviderOutboundRequestIdentityScope, ProviderOutboundRequestMutationScope,
|
||||
ProviderOutboundRequestPolicy, ProviderOutboundRequestPolicyOutcome,
|
||||
ProviderOutboundRequestPolicyReason, ProviderOutboundRequestPolicyResult,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
||||
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
||||
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
||||
@@ -121,5 +130,5 @@ pub(crate) use aether_provider_transport::{
|
||||
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
||||
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
||||
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
||||
GROK_RATE_LIMITS_PATH, WINDSURF_ENVELOPE_NAME,
|
||||
GROK_RATE_LIMITS_PATH, PROVIDER_OUTBOUND_CONTEXT_MAX_VALUE_BYTES, WINDSURF_ENVELOPE_NAME,
|
||||
};
|
||||
|
||||
@@ -53,6 +53,8 @@ const AI_ANY_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/operations/{*operation_path}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{*video_path}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{*video_path}",
|
||||
"/upload/v1beta/files",
|
||||
"/v1beta/files",
|
||||
"/v1beta/files/{*file_path}",
|
||||
|
||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
||||
"internal_gateway": {
|
||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||
"status": "rust_native_control_plane",
|
||||
"status": state.internal_gateway_auth_status(),
|
||||
},
|
||||
},
|
||||
"features": {
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Request, State};
|
||||
use axum::http::{self, HeaderValue, StatusCode};
|
||||
use axum::middleware::{self, Next};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use axum::{Json, Router};
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::async_task::{
|
||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
||||
use crate::router::metrics;
|
||||
use crate::state::AppState;
|
||||
|
||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
||||
router
|
||||
#[derive(Clone, Copy)]
|
||||
struct OperationalPermission {
|
||||
required_permissions: &'static [&'static str],
|
||||
write: bool,
|
||||
requires_full_admin_role: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn mount_operational_routes(
|
||||
router: Router<AppState>,
|
||||
state: AppState,
|
||||
) -> Router<AppState> {
|
||||
let operational = Router::<AppState>::new()
|
||||
.route("/_gateway/metrics", get(metrics))
|
||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||
.route(
|
||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
||||
"/_gateway/audit/request-usage/{request_id}",
|
||||
get(get_request_usage_audit),
|
||||
)
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
state,
|
||||
authorize_operational_request,
|
||||
));
|
||||
router.merge(operational)
|
||||
}
|
||||
|
||||
async fn authorize_operational_request(
|
||||
State(state): State<AppState>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response<Body> {
|
||||
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||
return operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational route permission is not configured",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let Some(remote_addr) = request
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|value| value.0)
|
||||
else {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let headers = request.headers().clone();
|
||||
let uri = request.uri().clone();
|
||||
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||
return operational_auth_required_response();
|
||||
}
|
||||
|
||||
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||
Ok(Some(principal)) => {
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||
&state, &headers, client_ip,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(authenticated) => authenticated,
|
||||
Err(
|
||||
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||
) => return operational_auth_required_response(),
|
||||
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
let missing_permission =
|
||||
permission
|
||||
.required_permissions
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|required| {
|
||||
!management_token_has_operational_permission(
|
||||
&authenticated.permissions,
|
||||
required,
|
||||
)
|
||||
});
|
||||
if let Some(required_permission) = missing_permission {
|
||||
return operational_permission_denied_response(required_permission);
|
||||
}
|
||||
|
||||
let client_ip = client_ip.to_string();
|
||||
if let Err(err) = state
|
||||
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
token_id = %authenticated.token.id,
|
||||
error = ?err,
|
||||
"gateway failed to record operational management token usage"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut response = next.run(request).await;
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||
if path == "/_gateway/metrics" {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: if write {
|
||||
&["admin:video_tasks:write"]
|
||||
} else {
|
||||
&["admin:video_tasks:read"]
|
||||
},
|
||||
write,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:api_keys:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &[
|
||||
"admin:monitoring:admin",
|
||||
"admin:usage:read",
|
||||
"admin:api_keys:read",
|
||||
],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||
{
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:admin"],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:usage:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn management_token_has_operational_permission(
|
||||
permissions: &[String],
|
||||
required_permission: &str,
|
||||
) -> bool {
|
||||
let scope = required_permission
|
||||
.rsplit_once(':')
|
||||
.map(|(scope, _)| scope)
|
||||
.unwrap_or(required_permission);
|
||||
let admin_permission = format!("{scope}:admin");
|
||||
permissions
|
||||
.iter()
|
||||
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||
}
|
||||
|
||||
fn operational_auth_required_response() -> Response<Body> {
|
||||
let mut response = operational_error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"admin authentication required",
|
||||
None,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
http::header::WWW_AUTHENTICATE,
|
||||
HeaderValue::from_static("Bearer"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||
operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational permission denied",
|
||||
Some(required_permission),
|
||||
)
|
||||
}
|
||||
|
||||
fn operational_error_response(
|
||||
status: StatusCode,
|
||||
detail: &'static str,
|
||||
required_permission: Option<&'static str>,
|
||||
) -> Response<Body> {
|
||||
let mut response = (
|
||||
status,
|
||||
Json(json!({
|
||||
"detail": detail,
|
||||
"required_permission": required_permission,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayLocalAuthRejection;
|
||||
use crate::headers::should_skip_response_header;
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use crate::{insert_header_if_missing, GatewayError};
|
||||
|
||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
||||
);
|
||||
}
|
||||
|
||||
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||
// Provider responses are API data, even when an untrusted provider labels
|
||||
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||
// from becoming same-origin active content, and prevent referrer leakage
|
||||
// if a user follows a link rendered from such a response.
|
||||
headers.insert(
|
||||
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("content-security-policy"),
|
||||
HeaderValue::from_static(
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||
),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("referrer-policy"),
|
||||
HeaderValue::from_static("no-referrer"),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response(
|
||||
upstream_response: reqwest::Response,
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let status = upstream_response.status();
|
||||
let upstream_headers = upstream_response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
(
|
||||
name.as_str().to_string(),
|
||||
value.to_str().unwrap_or_default().to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||
let upstream_stream = upstream_response.bytes_stream();
|
||||
build_client_response_from_parts(
|
||||
status.as_u16(),
|
||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
headers
|
||||
.get_all(http::header::CONNECTION)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok()),
|
||||
);
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
let normalized = name.as_str().to_ascii_lowercase();
|
||||
if should_skip_client_response_header(&normalized)
|
||||
|| connection_declared.contains(&normalized)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|value| (normalized, value.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn should_skip_client_response_header(name: &str) -> bool {
|
||||
should_skip_response_header(name)
|
||||
// A provider Location is relative to the provider, not to the gateway.
|
||||
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||
// can disclose their gateway Authorization header to another origin.
|
||||
// Keep Location available inside execution reports, but never expose
|
||||
// it on the client-facing response boundary.
|
||||
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response_from_parts(
|
||||
status_code: u16,
|
||||
upstream_headers: &BTreeMap<String, String>,
|
||||
@@ -111,8 +158,17 @@ where
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
upstream_headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||
.map(|(_, value)| value.as_str()),
|
||||
);
|
||||
|
||||
for (name, value) in upstream_headers {
|
||||
if should_skip_response_header(name.as_str()) {
|
||||
if should_skip_client_response_header(name.as_str())
|
||||
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||
@@ -123,6 +179,7 @@ where
|
||||
}
|
||||
mutate_headers(response.headers_mut())?;
|
||||
apply_streaming_response_headers(response.headers_mut());
|
||||
apply_gateway_browser_security_headers(response.headers_mut());
|
||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||
if let Some(decision) = control_decision {
|
||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_plan_usage_limited_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
rejection: &PlanUsagePolicyRejection,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||
let fallback_payload = json!({
|
||||
"error": {
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": message,
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
});
|
||||
let payload = build_local_error_payload(
|
||||
control_decision,
|
||||
None,
|
||||
message,
|
||||
LocalCoreSyncErrorKind::RateLimit,
|
||||
fallback_payload,
|
||||
);
|
||||
let body =
|
||||
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let headers = BTreeMap::from([
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||
(
|
||||
"X-RateLimit-Metric".to_string(),
|
||||
rejection.metric.to_string(),
|
||||
),
|
||||
(
|
||||
"X-RateLimit-Window".to_string(),
|
||||
rejection.window.to_string(),
|
||||
),
|
||||
]);
|
||||
build_client_response_from_parts(
|
||||
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||
&headers,
|
||||
Body::from(body),
|
||||
trace_id,
|
||||
control_decision,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_http_error_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
||||
build_client_response, build_client_response_from_parts,
|
||||
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||
build_local_user_rpm_limited_response,
|
||||
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||
};
|
||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use axum::body::{to_bytes, Body};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -490,6 +600,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||
let response = build_client_response_from_parts_with_mutator(
|
||||
200,
|
||||
&BTreeMap::from([
|
||||
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||
(
|
||||
"x-aether-gateway".to_string(),
|
||||
"attacker-gateway".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-control-action".to_string(),
|
||||
"attacker-action".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-future-control".to_string(),
|
||||
"attacker-future".to_string(),
|
||||
),
|
||||
(
|
||||
"x-accel-redirect".to_string(),
|
||||
"/internal/private-file".to_string(),
|
||||
),
|
||||
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||
(
|
||||
"x-reproxy-url".to_string(),
|
||||
"http://127.0.0.1:9000/private".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-origin".to_string(),
|
||||
"https://attacker.example".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-credentials".to_string(),
|
||||
"true".to_string(),
|
||||
),
|
||||
("content-length".to_string(), "999999".to_string()),
|
||||
(
|
||||
"content-security-policy".to_string(),
|
||||
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||
),
|
||||
(
|
||||
"content-security-policy-report-only".to_string(),
|
||||
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||
),
|
||||
(
|
||||
"reporting-endpoints".to_string(),
|
||||
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||
),
|
||||
("report-to".to_string(), "attacker".to_string()),
|
||||
(
|
||||
"nel".to_string(),
|
||||
"{\"report_to\":\"attacker\"}".to_string(),
|
||||
),
|
||||
(
|
||||
"refresh".to_string(),
|
||||
"0; url=https://attacker.example".to_string(),
|
||||
),
|
||||
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||
(
|
||||
"location".to_string(),
|
||||
"https://provider.example/direct".to_string(),
|
||||
),
|
||||
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||
]),
|
||||
Body::empty(),
|
||||
"trace-upstream-header-filter",
|
||||
None,
|
||||
|headers| {
|
||||
headers.insert(
|
||||
http::HeaderName::from_static("x-aether-control-action"),
|
||||
http::HeaderValue::from_static("gateway-action"),
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("response should build");
|
||||
|
||||
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||
assert!(response.headers().get("x-sendfile").is_none());
|
||||
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("content-security-policy-report-only")
|
||||
.is_none());
|
||||
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||
assert!(response.headers().get("report-to").is_none());
|
||||
assert!(response.headers().get("nel").is_none());
|
||||
assert!(response.headers().get("refresh").is_none());
|
||||
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||
assert_eq!(
|
||||
response.headers()["content-security-policy"],
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||
);
|
||||
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||
assert_eq!(
|
||||
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||
"nosniff"
|
||||
);
|
||||
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||
assert_eq!(
|
||||
response.headers()["x-aether-control-action"],
|
||||
"gateway-action"
|
||||
);
|
||||
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener");
|
||||
let addr = listener.local_addr().expect("address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).await.expect("request read");
|
||||
stream
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||
)
|
||||
.await
|
||||
.expect("response write");
|
||||
});
|
||||
let upstream = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("client")
|
||||
.get(format!("http://{addr}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upstream response");
|
||||
|
||||
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||
.expect("client response");
|
||||
server.await.expect("server");
|
||||
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-first-hop").is_none());
|
||||
assert!(response.headers().get("x-second-hop").is_none());
|
||||
}
|
||||
|
||||
fn claude_decision() -> GatewayControlDecision {
|
||||
GatewayControlDecision::synthetic(
|
||||
"/v1/messages",
|
||||
@@ -581,4 +848,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||
let response = build_local_plan_usage_limited_response(
|
||||
"trace-plan-limit",
|
||||
None,
|
||||
&PlanUsagePolicyRejection {
|
||||
metric: "request_count",
|
||||
limit: 100.0,
|
||||
retry_after: 42,
|
||||
window: "calendar_week",
|
||||
},
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(response.headers()["retry-after"], "42");
|
||||
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||
let payload = response_json(response).await;
|
||||
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_contracts::ExecutionResult;
|
||||
@@ -21,7 +22,9 @@ use super::{
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
||||
pub(crate) use self::cancel::{
|
||||
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ListVideoTasksQuery {
|
||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
||||
}
|
||||
|
||||
pub(crate) async fn build_video_task_video_response(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
task_id: &str,
|
||||
source: VideoTaskVideoSource,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
match source {
|
||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
||||
VideoTaskVideoSource::Redirect { url } => {
|
||||
resolve_public_video_target(&url).await?;
|
||||
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||
}
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
||||
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
||||
}
|
||||
|
||||
async fn proxy_video_stream(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
url: &str,
|
||||
url: &url::Url,
|
||||
header_name: &str,
|
||||
header_value: &str,
|
||||
filename: &str,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
let response = state
|
||||
.client
|
||||
.get(url)
|
||||
let target = resolve_public_video_target(url).await?;
|
||||
let client = build_pinned_video_client(&target)?;
|
||||
let response = client
|
||||
.get(url.clone())
|
||||
.header(header_name, header_value)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: err.to_string(),
|
||||
message: video_request_failure_message(&err).to_string(),
|
||||
})?;
|
||||
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
if response.status().is_redirection() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!(
|
||||
"video upstream redirect was rejected with HTTP {}",
|
||||
response.status()
|
||||
),
|
||||
});
|
||||
}
|
||||
if !response.status().is_success() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!("video upstream returned HTTP {}", response.status()),
|
||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
||||
let content_length = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_LENGTH)
|
||||
.cloned();
|
||||
let cache_control = response
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.cloned();
|
||||
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||
// Reqwest may decode transfer/content encodings and the provider controls
|
||||
// the declaration; forwarding a stale value would make the client-facing
|
||||
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||
// will select safe framing for the actual stream.
|
||||
let upstream_headers = response.headers().clone();
|
||||
let body = Body::from_stream(response.bytes_stream());
|
||||
|
||||
let mut outbound = axum::http::Response::builder()
|
||||
.status(status)
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
if let Some(content_length) = content_length {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
||||
}
|
||||
if let Some(cache_control) = cache_control {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
||||
} else {
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
||||
);
|
||||
}
|
||||
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||
Ok(outbound)
|
||||
}
|
||||
|
||||
fn apply_safe_video_response_metadata(
|
||||
outbound: &mut axum::http::HeaderMap,
|
||||
upstream: &axum::http::HeaderMap,
|
||||
filename: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
let content_type = upstream
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(safe_video_content_type)
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!(
|
||||
"inline; filename=\"{}\"",
|
||||
safe_video_filename(filename)
|
||||
))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||
outbound.insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
outbound.insert(
|
||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
axum::http::HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||
let subtype = media_type.strip_prefix("video/")?;
|
||||
if subtype.is_empty()
|
||||
|| !subtype.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||
)
|
||||
})
|
||||
{
|
||||
return None;
|
||||
}
|
||||
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||
}
|
||||
|
||||
fn safe_video_filename(filename: &str) -> String {
|
||||
let filename = filename
|
||||
.chars()
|
||||
.take(255)
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||
character
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
if filename.is_empty() {
|
||||
"video.mp4".to_string()
|
||||
} else {
|
||||
filename
|
||||
}
|
||||
}
|
||||
|
||||
struct ResolvedVideoTarget {
|
||||
host: String,
|
||||
addrs: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||
));
|
||||
}
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||
let (host, addrs) = match url.host() {
|
||||
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||
let addrs = aether_http::lookup_host_with_limits(
|
||||
host,
|
||||
port,
|
||||
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||
(host.to_string(), addrs)
|
||||
}
|
||||
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||
};
|
||||
if addrs.is_empty()
|
||||
|| addrs
|
||||
.iter()
|
||||
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL resolves to a private or reserved address",
|
||||
));
|
||||
}
|
||||
Ok(ResolvedVideoTarget { host, addrs })
|
||||
}
|
||||
|
||||
fn build_pinned_video_client(
|
||||
target: &ResolvedVideoTarget,
|
||||
) -> Result<reqwest::Client, GatewayError> {
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(10_000),
|
||||
request_timeout_ms: Some(300_000),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if target.host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||
}
|
||||
builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||
}
|
||||
|
||||
fn video_target_rejected(message: &str) -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"video upstream request timed out"
|
||||
} else if error.is_connect() {
|
||||
"video upstream connection failed"
|
||||
} else if error.is_body() || error.is_decode() {
|
||||
"video upstream response failed"
|
||||
} else {
|
||||
"video upstream request failed"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use super::{
|
||||
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_accepts_public_target() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let target = "https://8.8.8.8/video.mp4";
|
||||
|
||||
let response = build_video_task_video_response(
|
||||
&state,
|
||||
"task-public-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(target).expect("public target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("public redirect should build");
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(axum::http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(target)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
] {
|
||||
let error = build_video_task_video_response(
|
||||
&state,
|
||||
"task-rejected-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("private or reserved redirect target should be rejected");
|
||||
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
axum::http::StatusCode::BAD_GATEWAY,
|
||||
"unexpected status for {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_err(),
|
||||
"target should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||
for raw_url in [
|
||||
"https://8.8.8.8/video.mp4",
|
||||
"https://[2606:4700:4700::1111]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_ok(),
|
||||
"target should be accepted: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||
assert!(safe_video_content_type("video/mp4").is_some());
|
||||
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||
assert!(safe_video_content_type("video/").is_none());
|
||||
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||
assert!(safe_video_content_type("text/html").is_none());
|
||||
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||
assert_eq!(
|
||||
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||
"video_123.mp4___filename__attack.html"
|
||||
);
|
||||
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||
|
||||
let mut upstream = axum::http::HeaderMap::new();
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
axum::http::HeaderValue::from_static("text/html"),
|
||||
);
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_LENGTH,
|
||||
axum::http::HeaderValue::from_static("999999"),
|
||||
);
|
||||
let mut outbound = upstream.clone();
|
||||
apply_safe_video_response_metadata(
|
||||
&mut outbound,
|
||||
&upstream,
|
||||
"video.mp4\"; filename=\"attack.html",
|
||||
)
|
||||
.expect("video metadata should build");
|
||||
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("application/octet-stream")
|
||||
);
|
||||
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("nosniff")
|
||||
);
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::finalize_video_task_if_terminal;
|
||||
use super::super::read_video_task_detail;
|
||||
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||
use super::current_unix_secs;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
cancel_video_task_record_inner(state, task_id, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn cancel_video_task_record_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let user_id = user_id.trim();
|
||||
if user_id.is_empty() {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||
}
|
||||
|
||||
async fn cancel_video_task_record_inner(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
expected_user_id: Option<&str>,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let task = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(task) = task else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
|
||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
||||
}
|
||||
|
||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||
let mut finalize_mutation = None;
|
||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||
state
|
||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
||||
.await?;
|
||||
|
||||
let body_json = json!({});
|
||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
);
|
||||
let follow_up = if let Some(user_id) = expected_user_id {
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
user_id,
|
||||
task.api_key_id.as_deref(),
|
||||
&trace_id,
|
||||
)
|
||||
} else {
|
||||
state
|
||||
.hydrate_video_task_for_route(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
)
|
||||
.await?;
|
||||
state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(follow_up) = follow_up {
|
||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||
.await
|
||||
.map_err(CancelVideoTaskError::Response)?;
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
} else if expected_user_id.is_none() {
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
||||
}
|
||||
|
||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||
Some(stored) => stored,
|
||||
None => {
|
||||
let current = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(current) = current else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
if !current.status.is_active() {
|
||||
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||
}
|
||||
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
"video task repository is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
)));
|
||||
}
|
||||
};
|
||||
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&request_path, &report_kind);
|
||||
}
|
||||
finalize_video_task_if_terminal(state, &stored).await;
|
||||
Ok(stored)
|
||||
}
|
||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
||||
}
|
||||
|
||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||
let provider_api_format = task
|
||||
.provider_api_format
|
||||
.as_deref()
|
||||
.or(task.client_api_format.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let provider_api_format = task.effective_api_format()?;
|
||||
|
||||
match provider_api_format {
|
||||
"openai:video" => Some(VideoTaskCancelPlan {
|
||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
||||
let result =
|
||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
.map_err(|_| {
|
||||
GatewayError::UpstreamUnavailable {
|
||||
trace_id: trace_id.to_string(),
|
||||
message: format!("{err:?}"),
|
||||
message: "video cancellation request failed".to_string(),
|
||||
}
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
if result.status_code >= 400 {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
let body_json = result
|
||||
.body
|
||||
.and_then(|body| body.json_body)
|
||||
.unwrap_or_else(|| {
|
||||
json!({
|
||||
"error": {
|
||||
"message": result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.unwrap_or_else(|| {
|
||||
format!("execution runtime returned {}", result.status_code)
|
||||
}),
|
||||
}
|
||||
})
|
||||
});
|
||||
return Err((status, Json(body_json)).into_response());
|
||||
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_cancelled_request_metadata(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<Value>, GatewayError> {
|
||||
let mut metadata = match task.request_metadata.clone() {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
||||
if snapshot_value.is_none() {
|
||||
snapshot_value = state
|
||||
.reconstruct_video_task_snapshot(task)
|
||||
.await?
|
||||
.map(|snapshot| {
|
||||
serde_json::to_value(snapshot)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
}
|
||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
snapshot_value_ref.clone(),
|
||||
);
|
||||
return Ok(Some(Value::Object(metadata)));
|
||||
}
|
||||
|
||||
Ok(task.request_metadata.clone())
|
||||
}
|
||||
|
||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("OpenAi")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
return;
|
||||
}
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("Gemini")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
}
|
||||
fn build_video_task_cancel_upstream_error_response(
|
||||
result: &aether_contracts::ExecutionResult,
|
||||
) -> axum::response::Response {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
tracing::warn!(
|
||||
event_name = "video_task_cancel_upstream_error",
|
||||
upstream_status = result.status_code,
|
||||
"video cancellation upstream response body discarded"
|
||||
);
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": format!(
|
||||
"video cancellation upstream returned HTTP {}",
|
||||
result.status_code
|
||||
),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn persist_cancelled_video_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
request_metadata: Option<Value>,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
let now_unix_secs = current_unix_secs();
|
||||
state
|
||||
.data
|
||||
.upsert_video_task(UpsertVideoTask {
|
||||
.update_active_video_task(UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
request_id: task.request_id.clone(),
|
||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: task.original_request_body.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: VideoTaskStatus::Cancelled,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds,
|
||||
next_poll_at_unix_secs: None,
|
||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
||||
completed_at_unix_secs: Some(now_unix_secs),
|
||||
updated_at_unix_secs: now_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata,
|
||||
request_metadata: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||
};
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
|
||||
use super::build_video_task_cancel_upstream_error_response;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "cancel-secret-request-id".to_string(),
|
||||
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||
status_code: 502,
|
||||
headers: BTreeMap::from([(
|
||||
"x-internal-secret".to_string(),
|
||||
"cancel-secret-header".to_string(),
|
||||
)]),
|
||||
response_observation: None,
|
||||
body: Some(ResponseBody {
|
||||
json_body: Some(json!({
|
||||
"error": {
|
||||
"message": "cancel-secret-upstream-body",
|
||||
}
|
||||
})),
|
||||
body_bytes_b64: None,
|
||||
}),
|
||||
telemetry: None,
|
||||
error: Some(ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::FirstByte,
|
||||
message: "cancel-secret-runtime-error".to_string(),
|
||||
upstream_status: Some(502),
|
||||
retryable: true,
|
||||
failover_recommended: false,
|
||||
}),
|
||||
};
|
||||
|
||||
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||
assert!(response.headers().get("x-internal-secret").is_none());
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("response body should parse");
|
||||
|
||||
assert_eq!(
|
||||
payload,
|
||||
json!({
|
||||
"error": {
|
||||
"message": "video cancellation upstream returned HTTP 502",
|
||||
}
|
||||
})
|
||||
);
|
||||
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||
assert!(!body.contains("cancel-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||
pub(crate) use http::{
|
||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
||||
CancelVideoTaskError,
|
||||
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||
};
|
||||
pub(crate) use query::{
|
||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
||||
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
pub(crate) use runtime::{
|
||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||
|
||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
||||
pub(crate) processing_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum VideoTaskVideoSource {
|
||||
Redirect {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
},
|
||||
Proxy {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
header_name: String,
|
||||
header_value: String,
|
||||
filename: String,
|
||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
||||
state.find_video_task_by_id(task_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_detail_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_video_source(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
video_task_video_source_from_task(state, &task).await
|
||||
}
|
||||
|
||||
pub(crate) async fn video_task_video_source_from_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||
let Some(video_url) = task
|
||||
.video_url
|
||||
.as_deref()
|
||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
||||
let video_url = parse_video_url(&video_url)?;
|
||||
|
||||
if task.effective_api_format() != Some("gemini:video") {
|
||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||
}
|
||||
|
||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
||||
));
|
||||
};
|
||||
|
||||
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||
})?;
|
||||
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||
});
|
||||
}
|
||||
let api_key = transport.key.decrypted_api_key.trim();
|
||||
if api_key.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
||||
url: video_url,
|
||||
header_name: "x-goog-api-key".to_string(),
|
||||
header_value: api_key.to_string(),
|
||||
filename: format!("video_{task_id}.mp4"),
|
||||
filename: format!("video_{}.mp4", task.id),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL is invalid".to_string(),
|
||||
})?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||
left.scheme() == right.scheme()
|
||||
&& left.host() == right.host()
|
||||
&& left.port_or_known_default() == right.port_or_known_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_stats(
|
||||
state: &AppState,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||
now_unix_secs - (now_unix_secs % 86_400)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::{data::GatewayDataState, AppState};
|
||||
|
||||
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||
{
|
||||
UpsertVideoTask {
|
||||
id: "legacy-gemini-task".to_string(),
|
||||
short_id: Some("legacy-short".to_string()),
|
||||
request_id: "legacy-request".to_string(),
|
||||
user_id: Some("user-1".to_string()),
|
||||
api_key_id: Some("client-key-1".to_string()),
|
||||
username: None,
|
||||
api_key_name: None,
|
||||
external_task_id: Some("operations/upstream-1".to_string()),
|
||||
provider_id: Some("provider-1".to_string()),
|
||||
endpoint_id: Some("endpoint-1".to_string()),
|
||||
key_id: Some("provider-key-1".to_string()),
|
||||
client_api_format: Some("gemini:video".to_string()),
|
||||
provider_api_format: None,
|
||||
format_converted: false,
|
||||
model: Some("veo-3".to_string()),
|
||||
prompt: None,
|
||||
original_request_body: None,
|
||||
duration_seconds: Some(8),
|
||||
resolution: Some("720p".to_string()),
|
||||
aspect_ratio: Some("16:9".to_string()),
|
||||
size: Some("1280x720".to_string()),
|
||||
status: VideoTaskStatus::Completed,
|
||||
progress_percent: 100,
|
||||
progress_message: None,
|
||||
retry_count: 0,
|
||||
poll_interval_seconds: 10,
|
||||
next_poll_at_unix_secs: None,
|
||||
poll_count: 1,
|
||||
max_poll_count: 360,
|
||||
created_at_unix_ms: 1,
|
||||
submitted_at_unix_secs: Some(1),
|
||||
completed_at_unix_secs: Some(2),
|
||||
updated_at_unix_secs: 2,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
video_url: Some(
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
.to_string(),
|
||||
),
|
||||
request_metadata: None,
|
||||
}
|
||||
.into_stored()
|
||||
}
|
||||
|
||||
fn state_with_gemini_transport() -> AppState {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let provider = StoredProviderCatalogProvider::new(
|
||||
"provider-1".to_string(),
|
||||
"Gemini".to_string(),
|
||||
Some("https://ai.google.dev".to_string()),
|
||||
"gemini".to_string(),
|
||||
)
|
||||
.expect("provider should build");
|
||||
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||
"endpoint-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"gemini:video".to_string(),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("endpoint should build")
|
||||
.with_transport_fields(
|
||||
"https://generativelanguage.googleapis.com".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let encrypted_api_key = state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-1",
|
||||
"provider-key-1",
|
||||
"gemini-provider-secret",
|
||||
)
|
||||
.expect("provider key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
"provider-key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"default".to_string(),
|
||||
"api_key".to_string(),
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("provider key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["gemini:video"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("provider key transport should build");
|
||||
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||
);
|
||||
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||
video_tasks,
|
||||
provider_catalog,
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
state.with_data_state_for_tests(data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||
for raw_url in [
|
||||
"file:///etc/passwd",
|
||||
"data:video/mp4;base64,AAAA",
|
||||
"https://[email protected]/video.mp4",
|
||||
"https://user:[email protected]/video.mp4",
|
||||
"/relative/video.mp4",
|
||||
] {
|
||||
assert!(
|
||||
parse_video_url(raw_url).is_err(),
|
||||
"URL should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||
for same_origin in [
|
||||
"https://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:443/file",
|
||||
] {
|
||||
assert!(video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(same_origin).unwrap()
|
||||
));
|
||||
}
|
||||
for different_origin in [
|
||||
"http://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:444/file",
|
||||
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||
] {
|
||||
assert!(!video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(different_origin).unwrap()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||
let source = video_task_video_source_from_task(
|
||||
&state_with_gemini_transport(),
|
||||
&legacy_gemini_video_task(),
|
||||
)
|
||||
.await
|
||||
.expect("video source should resolve")
|
||||
.expect("video source should exist");
|
||||
|
||||
match source {
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => {
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
);
|
||||
assert_eq!(header_name, "x-goog-api-key");
|
||||
assert_eq!(header_value, "gemini-provider-secret");
|
||||
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||
}
|
||||
VideoTaskVideoSource::Redirect { .. } => {
|
||||
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VideoTaskRefreshError {
|
||||
message: String,
|
||||
category: &'static str,
|
||||
permanent: bool,
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
||||
warn!(
|
||||
event_name = "video_task_refresh_failed",
|
||||
log_type = "event",
|
||||
error = %err.message,
|
||||
error_category = err.category,
|
||||
permanent = err.permanent,
|
||||
"gateway video task refresh failed"
|
||||
);
|
||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let mut refreshed = 0usize;
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
let trace_id = format!("video-task-poller-{index}");
|
||||
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||
continue;
|
||||
};
|
||||
let Some(refresh_plan) = state
|
||||
.video_tasks
|
||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
||||
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||
let Some(updated) =
|
||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
||||
let Some(updated) = build_successful_poll_update(
|
||||
&task,
|
||||
snapshot.clone(),
|
||||
&provider_body,
|
||||
now_unix_secs,
|
||||
)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: format!("{err:?}"),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
}));
|
||||
}
|
||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.and_then(|body| body.as_object().cloned())
|
||||
else {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: "video task refresh missing json provider body".to_string(),
|
||||
category: "invalid_provider_response",
|
||||
permanent: false,
|
||||
}));
|
||||
};
|
||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
.as_ref()
|
||||
.and_then(|error| error.upstream_status)
|
||||
.unwrap_or(result.status_code);
|
||||
let message = result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.or_else(|| {
|
||||
result
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|body| body.json_body.as_ref())
|
||||
.and_then(|value| value.get("error"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
||||
let category = if status_code == 401 {
|
||||
"authentication_error"
|
||||
} else if status_code == 403 {
|
||||
"permission_denied"
|
||||
} else if status_code == 404 {
|
||||
"not_found"
|
||||
} else if status_code == 429 {
|
||||
"rate_limit"
|
||||
} else if status_code >= 500 {
|
||||
"server_error"
|
||||
} else {
|
||||
"provider_error"
|
||||
};
|
||||
let permanent = result.error.as_ref().map_or(
|
||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||
|error| match error.kind {
|
||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
},
|
||||
);
|
||||
|
||||
VideoTaskRefreshError { message, permanent }
|
||||
VideoTaskRefreshError {
|
||||
category,
|
||||
permanent,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_successful_poll_update(
|
||||
task: &StoredVideoTask,
|
||||
mut snapshot: LocalVideoTaskSnapshot,
|
||||
provider_body: &Map<String, Value>,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
||||
return Ok(None);
|
||||
};
|
||||
snapshot.apply_provider_body(provider_body);
|
||||
|
||||
let mut record = snapshot.to_upsert_record();
|
||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
||||
record.format_converted = task.format_converted;
|
||||
record.model = task.model.clone().or(record.model);
|
||||
record.prompt = task.prompt.clone().or(record.prompt);
|
||||
record.original_request_body = task
|
||||
.original_request_body
|
||||
.clone()
|
||||
.or(record.original_request_body);
|
||||
record.original_request_body = None;
|
||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||
record.resolution = task.resolution.clone().or(record.resolution);
|
||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
Some(provider_body),
|
||||
None,
|
||||
)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
record.request_metadata = None;
|
||||
|
||||
Ok(Some(record))
|
||||
}
|
||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
||||
let mut record = stored_task_to_upsert(task);
|
||||
record.updated_at_unix_secs = now_unix_secs;
|
||||
record.poll_count = task.poll_count.saturating_add(1);
|
||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
||||
record.progress_message = None;
|
||||
if err.permanent {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_permanent_error".to_string());
|
||||
record.error_message = Some(err.message.clone());
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
} else {
|
||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| {
|
||||
merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
None,
|
||||
Some(err),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or(task.request_metadata.clone());
|
||||
record.request_metadata = None;
|
||||
record
|
||||
}
|
||||
|
||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
let snapshot_record =
|
||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
||||
UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
provider_api_format: task.provider_api_format.clone(),
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.prompt.clone())
|
||||
}),
|
||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.original_request_body.clone())
|
||||
}),
|
||||
duration_seconds: task.duration_seconds.or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.duration_seconds)
|
||||
}),
|
||||
resolution: task.resolution.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.resolution.clone())
|
||||
}),
|
||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.aspect_ratio.clone())
|
||||
}),
|
||||
size: task.size.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.size.clone())
|
||||
}),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: task.status,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||
}
|
||||
|
||||
fn merge_video_task_request_metadata(
|
||||
existing: Option<Value>,
|
||||
snapshot: &LocalVideoTaskSnapshot,
|
||||
provider_body: Option<&Map<String, Value>>,
|
||||
poll_error: Option<&VideoTaskRefreshError>,
|
||||
) -> Result<Option<Value>, serde_json::Error> {
|
||||
let mut metadata = match existing {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
serde_json::to_value(snapshot)?,
|
||||
);
|
||||
if let Some(provider_body) = provider_body {
|
||||
metadata.insert(
|
||||
"poll_raw_response".to_string(),
|
||||
Value::Object(provider_body.clone()),
|
||||
);
|
||||
metadata.remove("poll_error");
|
||||
}
|
||||
if let Some(poll_error) = poll_error {
|
||||
metadata.insert(
|
||||
"poll_error".to_string(),
|
||||
serde_json::json!({
|
||||
"message": poll_error.message,
|
||||
"permanent": poll_error.permanent,
|
||||
"observed_at_unix_secs": now_unix_secs(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(Some(Value::Object(metadata)))
|
||||
}
|
||||
|
||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||
return;
|
||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
||||
.or_else(|| task.provider_id.clone())
|
||||
let provider_name = task
|
||||
.provider_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let response_time_ms = task
|
||||
.submitted_at_unix_secs
|
||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
has_format_conversion: Some(task.format_converted),
|
||||
is_stream: Some(false),
|
||||
status_code,
|
||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
||||
error_message: task.error_code.clone(),
|
||||
response_time_ms,
|
||||
request_body: task.original_request_body.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_body: None,
|
||||
request_metadata: None,
|
||||
..UsageEventData::default()
|
||||
},
|
||||
))
|
||||
@@ -609,6 +536,9 @@ mod tests {
|
||||
|
||||
fn sample_sparse_stored_task() -> StoredVideoTask {
|
||||
let snapshot = LocalVideoTaskSnapshot::OpenAi(OpenAiVideoTaskSeed {
|
||||
local_short_id: None,
|
||||
native_response: None,
|
||||
xai_provider: false,
|
||||
local_task_id: "task-1".to_string(),
|
||||
upstream_task_id: "ext-1".to_string(),
|
||||
created_at_unix_ms: 1,
|
||||
@@ -701,48 +631,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
||||
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.duration_seconds, Some(4));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.duration_seconds.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.aspect_ratio.is_none());
|
||||
assert!(record.size.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
||||
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||
let record = build_failed_poll_update(
|
||||
&sample_sparse_stored_task(),
|
||||
&VideoTaskRefreshError {
|
||||
message: "temporary failure".to_string(),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
},
|
||||
100,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
||||
path_and_query: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) {
|
||||
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||
let Some(decision) = control_decision else {
|
||||
return;
|
||||
};
|
||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
||||
},
|
||||
route_kind,
|
||||
default_target_type(route_family),
|
||||
path_and_query.to_string(),
|
||||
sanitized_path_and_query.clone(),
|
||||
)
|
||||
};
|
||||
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||
|
||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||
if log_level == AdminAuditLogLevel::Info {
|
||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||
if target_id.trim_start().starts_with('/') {
|
||||
return sanitize_admin_audit_path(&target_id);
|
||||
}
|
||||
target_id
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AdminAuditLogLevel {
|
||||
Info,
|
||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
||||
use super::{
|
||||
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||
AdminAuditLogLevel,
|
||||
};
|
||||
use axum::http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
@@ -169,4 +185,32 @@ mod tests {
|
||||
("failed", AdminAuditLogLevel::Warn)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_paths_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path(
|
||||
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||
),
|
||||
"/api/admin/providers?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||
"/install/[redacted]?view=raw"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id(
|
||||
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||
),
|
||||
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||
"resource-id?literal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
|
||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||
use super::scopes::BackupScope;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct S3BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) scope: BackupScope,
|
||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
||||
pub(crate) retention_count: u32,
|
||||
}
|
||||
|
||||
impl fmt::Debug for S3BackupConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||
formatter
|
||||
.debug_struct("S3BackupConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field("scope", &self.scope)
|
||||
.field("endpoint_origin", &endpoint_origin)
|
||||
.field("region", &self.region)
|
||||
.field("user_agent", &self.user_agent)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||
.field("path_style", &self.path_style)
|
||||
.field("compression", &self.compression)
|
||||
.field("schedule", &self.schedule)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct BackupConfigError {
|
||||
message: String,
|
||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
||||
"Endpoint(S3 地址)",
|
||||
enabled,
|
||||
)?;
|
||||
if enabled {
|
||||
validate_s3_endpoint(&endpoint)?;
|
||||
}
|
||||
let bucket =
|
||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||
let access_key_id = required_or_disabled_string(
|
||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
||||
enabled,
|
||||
)?;
|
||||
|
||||
let prefix = normalize_s3_prefix(
|
||||
&optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
scope,
|
||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||
bucket,
|
||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
prefix,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||
let prefix = prefix.trim().trim_matches('/');
|
||||
if prefix.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
if prefix
|
||||
.split('/')
|
||||
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||
|| prefix.contains('\\')
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||
let parsed = url::Url::parse(endpoint)
|
||||
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||
if parsed.scheme() != "https"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||
url::Url::parse(endpoint)
|
||||
.ok()
|
||||
.map(|parsed| parsed.origin().ascii_serialization())
|
||||
.unwrap_or_else(|| "<invalid>".to_string())
|
||||
}
|
||||
|
||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||
if (min..=max).contains(&value) {
|
||||
Ok(())
|
||||
@@ -374,6 +445,69 @@ mod tests {
|
||||
assert!(err.to_string().contains("Endpoint"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||
for endpoint in [
|
||||
"http://s3.example.com",
|
||||
"https://user:[email protected]",
|
||||
"https://s3.example.com?token=secret",
|
||||
"https://s3.example.com/#fragment",
|
||||
] {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": endpoint,
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
|
||||
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect_err("unsafe endpoint should fail closed");
|
||||
assert!(error.to_string().contains("Endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_output_does_not_expose_s3_credentials() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access-key-value",
|
||||
"backup_s3_secret_access_key": "secret-key-value"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("config should parse");
|
||||
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains("https://s3.example.com"));
|
||||
assert!(!debug.contains("/path"));
|
||||
assert!(!debug.contains("access-key-value"));
|
||||
assert!(!debug.contains("secret-key-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_s3_backup_prefix_once() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_prefix": "/prod/backups//",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("prefix should be canonicalized");
|
||||
|
||||
assert_eq!(config.prefix, "prod/backups/");
|
||||
|
||||
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||
let mut entries = entries.clone();
|
||||
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_default_values_from_system_config_contract() {
|
||||
let entries = serde_json::json!({
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
||||
pub(crate) mod task;
|
||||
pub(crate) mod worker;
|
||||
|
||||
pub use executor::{
|
||||
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||
};
|
||||
|
||||
use axum::body::Bytes;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupRestoreScope {
|
||||
Config,
|
||||
Users,
|
||||
Data,
|
||||
}
|
||||
|
||||
impl BackupRestoreScope {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "config",
|
||||
Self::Users => "users",
|
||||
Self::Data => "data",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("backup database apply failed: {0}")]
|
||||
pub struct BackupApplyError(String);
|
||||
|
||||
pub async fn apply_restored_backup(
|
||||
app: &crate::AppState,
|
||||
restored: RestoredBackupJson,
|
||||
scope: BackupRestoreScope,
|
||||
operator_id: Option<&str>,
|
||||
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||
if authority.scope() != scope {
|
||||
return Err(BackupApplyError(format!(
|
||||
"authenticated {} backup cannot be applied to {} scope",
|
||||
authority.scope().as_str(),
|
||||
scope.as_str(),
|
||||
)));
|
||||
}
|
||||
let request_body = Bytes::from(json_bytes);
|
||||
let state = crate::admin_api::AdminAppState::new(app);
|
||||
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupRestoreScope::Config => {
|
||||
state
|
||||
.restore_admin_system_config_backup(&request_body, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Users => {
|
||||
state
|
||||
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Data => {
|
||||
state
|
||||
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error {
|
||||
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||
"another system import or restore is already running"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||
"system import coordination is unavailable"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||
}
|
||||
};
|
||||
BackupApplyError(message.to_string())
|
||||
})?;
|
||||
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||
}
|
||||
|
||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||
RestoredBackupJson,
|
||||
};
|
||||
use crate::backup::executor::encrypt_backup_bytes;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use serde_json::json;
|
||||
|
||||
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||
let compressed = zstd::stream::encode_all(
|
||||
serde_json::to_vec(&json!({
|
||||
"version": "1.5",
|
||||
"exported_at": "2026-08-30T12:00:00Z",
|
||||
"users": [],
|
||||
"standalone_keys": [],
|
||||
}))
|
||||
.expect("test backup should serialize")
|
||||
.as_slice(),
|
||||
0,
|
||||
)
|
||||
.expect("test backup should compress");
|
||||
let (envelope, _) =
|
||||
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||
.expect("test backup should encrypt");
|
||||
super::restore_backup_json(
|
||||
object_key,
|
||||
&envelope,
|
||||
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.expect("test restore key should build")],
|
||||
BackupRestoreLimits::default(),
|
||||
)
|
||||
.expect("test backup should authenticate")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||
let restored = authenticated_users_backup();
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&crate::AppState::new().expect("test state should build"),
|
||||
restored,
|
||||
BackupRestoreScope::Config,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("scope mismatch must fail before database access");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||
let app = crate::AppState::new().expect("test state should build");
|
||||
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the shared import lease");
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&app,
|
||||
authenticated_users_backup(),
|
||||
BackupRestoreScope::Users,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("restore must not interleave with another system import");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: another system import or restore is already running"
|
||||
);
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::fmt;
|
||||
|
||||
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupScope {
|
||||
Config,
|
||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||
if object_key.is_empty()
|
||||
|| object_key.starts_with('/')
|
||||
|| object_key.contains('\0')
|
||||
|| object_key.contains('\\')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut segments = object_key.split('/').peekable();
|
||||
let mut file_name = None;
|
||||
while let Some(segment) = segments.next() {
|
||||
if segment.is_empty()
|
||||
|| segment == "."
|
||||
|| segment == ".."
|
||||
|| segment.chars().any(char::is_control)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if segments.peek().is_none() {
|
||||
file_name = Some(segment);
|
||||
}
|
||||
}
|
||||
let file_name = file_name?;
|
||||
|
||||
[Self::Config, Self::Users, Self::Data]
|
||||
.into_iter()
|
||||
.find(|scope| {
|
||||
file_name
|
||||
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||
.is_some_and(is_aether_backup_object_id)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn matching_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[
|
||||
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn matching_encrypted_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||
}
|
||||
|
||||
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||
)
|
||||
}
|
||||
|
||||
fn matching_backup_keys_with_suffixes(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
file_suffixes: &[&str],
|
||||
) -> Vec<String> {
|
||||
let normalized_prefix = normalized_prefix(prefix);
|
||||
let expected_prefix = if normalized_prefix.is_empty() {
|
||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
||||
format!("{normalized_prefix}/")
|
||||
};
|
||||
let file_prefix = format!("{}-", self.file_stem());
|
||||
let file_suffix = ".json.zst";
|
||||
|
||||
keys.into_iter()
|
||||
.filter(|key| {
|
||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
||||
if file_name.contains('/') {
|
||||
return false;
|
||||
}
|
||||
let Some(timestamp) = file_name
|
||||
.strip_prefix(&file_prefix)
|
||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
||||
else {
|
||||
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||
file_suffixes
|
||||
.iter()
|
||||
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(timestamp)
|
||||
is_aether_backup_object_id(timestamp)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn file_name(self, timestamp: &str) -> String {
|
||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
||||
format!(
|
||||
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||
self.file_stem()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||
if is_aether_backup_timestamp(value) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||
let (time, digest) = rest.split_once('-')?;
|
||||
Some((format!("{date}-{time}"), digest))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(×tamp)
|
||||
&& collision_digest.len() == 64
|
||||
&& collision_digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::BackupScope;
|
||||
@@ -130,15 +226,15 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
||||
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
||||
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -146,7 +242,7 @@ mod tests {
|
||||
fn retention_filter_only_matches_same_scope() {
|
||||
let keys = vec![
|
||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/random.json.zst".to_string(),
|
||||
];
|
||||
@@ -155,14 +251,18 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_filter_requires_aether_timestamp_format() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
let keys = vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||
@@ -171,13 +271,19 @@ mod tests {
|
||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||
];
|
||||
|
||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,11 +291,11 @@ mod tests {
|
||||
fn backup_key_prefix_boundaries_are_exact() {
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("", "20260524-031500"),
|
||||
"aether-config-backup-20260524-031500.json.zst"
|
||||
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
|
||||
let keys = vec![
|
||||
@@ -208,4 +314,36 @@ mod tests {
|
||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(
|
||||
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||
),
|
||||
Some(BackupScope::Config)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(&format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
)),
|
||||
Some(BackupScope::Users)
|
||||
);
|
||||
for key in [
|
||||
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||
] {
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(key),
|
||||
None,
|
||||
"unsafe or unrelated key: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::TryStreamExt;
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use object_store::path::Path;
|
||||
use object_store::{ClientOptions, ObjectStore};
|
||||
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||
use reqwest::header::HeaderValue;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupObjectCreateResult {
|
||||
Created,
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError>;
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let mut objects = self.objects.write().await;
|
||||
if objects.contains_key(key) {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
} else {
|
||||
objects.insert(key.to_string(), bytes);
|
||||
Ok(BackupObjectCreateResult::Created)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let bytes = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.get(key)
|
||||
.cloned()
|
||||
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||
if bytes.len() > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix = directory_list_prefix(prefix);
|
||||
Ok(self
|
||||
let keys: Vec<_> = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.starts_with(&prefix))
|
||||
.cloned()
|
||||
.collect())
|
||||
.collect();
|
||||
if keys.len() > max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
#[cfg(test)]
|
||||
impl FakeBackupObjectStore {
|
||||
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||
self.objects.read().await.get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut keys = self
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let options = PutOptions {
|
||||
mode: PutMode::Create,
|
||||
..PutOptions::default()
|
||||
};
|
||||
match self
|
||||
.store
|
||||
.list(prefix_path.as_ref())
|
||||
.map_ok(|meta| meta.location.to_string())
|
||||
.try_collect::<Vec<_>>()
|
||||
.put_opts(&Path::from(key), bytes.into(), options)
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
{
|
||||
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
}
|
||||
Err(error) => Err(BackupStoreError::object_store(
|
||||
"conditional put",
|
||||
key,
|
||||
error,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let result = self
|
||||
.store
|
||||
.get(&Path::from(key))
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
let object_size = result.meta.size;
|
||||
let mut stream = result.into_stream();
|
||||
let mut bytes = BytesMut::with_capacity(
|
||||
usize::try_from(object_size)
|
||||
.unwrap_or(max_bytes)
|
||||
.min(max_bytes)
|
||||
.min(8 * 1024 * 1024),
|
||||
);
|
||||
while let Some(chunk) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes.freeze())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut objects = self.store.list(prefix_path.as_ref());
|
||||
let mut keys = Vec::new();
|
||||
while let Some(meta) = objects
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||
{
|
||||
if keys.len() >= max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
keys.push(meta.location.to_string());
|
||||
}
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_list_prefix(prefix: &str) -> String {
|
||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
||||
use super::{
|
||||
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
||||
async fn fake_backup_object_store_puts_and_lists() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object(
|
||||
@@ -186,17 +336,59 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
assert_eq!(keys.len(), 2);
|
||||
|
||||
store
|
||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||
assert_eq!(
|
||||
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||
bytes::Bytes::from_static(b"1234")
|
||||
);
|
||||
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::Created
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::AlreadyExists
|
||||
);
|
||||
assert_eq!(
|
||||
store.object_bytes(key).await.as_deref(),
|
||||
Some(b"first".as_slice())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -218,7 +410,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
keys,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_admin::system::admin_system_config_default_value;
|
||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
||||
use futures_util::FutureExt;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::warn;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||
use super::scopes::BackupScope;
|
||||
use super::store::ObjectStoreS3BackupStore;
|
||||
use crate::admin_api::AdminAppState;
|
||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
||||
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||
use crate::task_runtime::{
|
||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
||||
];
|
||||
|
||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
enum BackupLockRenewalFailure<E> {
|
||||
Lost,
|
||||
Backend(E),
|
||||
}
|
||||
|
||||
enum BackupLockRaceOutcome<T> {
|
||||
BackupCompleted(T),
|
||||
LeaseLost(Result<(), JoinError>),
|
||||
}
|
||||
|
||||
impl S3BackupTaskError {
|
||||
fn bad_request(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
||||
impl std::error::Error for S3BackupTaskError {}
|
||||
|
||||
impl From<GatewayError> for S3BackupTaskError {
|
||||
fn from(error: GatewayError) -> Self {
|
||||
Self::internal(format!("{error:?}"))
|
||||
fn from(_error: GatewayError) -> Self {
|
||||
warn!(
|
||||
error_category = "dependency_failed",
|
||||
"S3 backup dependency failed"
|
||||
);
|
||||
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
||||
) -> Value {
|
||||
let mut payload = json!({
|
||||
"scope": config.scope.as_config_value(),
|
||||
"bucket": config.bucket.clone(),
|
||||
"prefix": config.prefix.clone(),
|
||||
"compression": config.compression.clone(),
|
||||
"trigger": trigger,
|
||||
});
|
||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
||||
Some(100),
|
||||
Some("S3 备份任务异常退出".to_string()),
|
||||
None,
|
||||
Some("S3 backup task panicked".to_string()),
|
||||
Some("background_task_panicked".to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
||||
.await;
|
||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
||||
let result = run_s3_backup_once(&app, &config).await;
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||
.await
|
||||
{
|
||||
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => result,
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task lost its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||
match heartbeat_result {
|
||||
Ok(()) => warn!(
|
||||
run_id = %run_id,
|
||||
"S3 backup task stopped after losing its distributed lock"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
run_id = %run_id,
|
||||
error = %error,
|
||||
"S3 backup lock heartbeat task failed"
|
||||
),
|
||||
}
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务已停止",
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(result) => {
|
||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
||||
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||
warn!(
|
||||
error_category = "slot_record_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup slot record failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||
None,
|
||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
||||
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup slot record failed",
|
||||
Some(json!({ "error": format!("{error:?}") })),
|
||||
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup task failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务失败".to_string()),
|
||||
None,
|
||||
Some(error.to_string()),
|
||||
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup task failed",
|
||||
Some(json!({ "error": error.to_string() })),
|
||||
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||
backup: F,
|
||||
mut heartbeat: JoinHandle<()>,
|
||||
) -> BackupLockRaceOutcome<T>
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
tokio::pin!(backup);
|
||||
tokio::select! {
|
||||
biased;
|
||||
heartbeat_result = &mut heartbeat => {
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||
}
|
||||
result = &mut backup => {
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
BackupLockRaceOutcome::BackupCompleted(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn require_successful_backup_lock_renewal<E>(
|
||||
result: Result<bool, E>,
|
||||
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||
match result {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_s3_backup_task_heartbeat(
|
||||
app: AppState,
|
||||
run_id: String,
|
||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let _ = app
|
||||
.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await;
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task distributed lock is no longer owned"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task distributed lock renewal failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||
)),
|
||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
||||
"无法获取 S3 备份任务锁:{error}"
|
||||
))),
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "lock_acquisition_failed",
|
||||
"S3 backup task lock acquisition failed"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法获取 S3 备份任务锁,请稍后重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
||||
app: &AppState,
|
||||
config: &S3BackupConfig,
|
||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
let payload = match config.scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
||||
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||
return Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||
));
|
||||
};
|
||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now())
|
||||
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||
warn!(
|
||||
error_category = "object_store_initialization_failed",
|
||||
"S3 backup object store initialization failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||
.await
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
||||
.map_err(|_| {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
"S3 backup execution failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})
|
||||
}
|
||||
|
||||
async fn build_s3_backup_payload_exclusively(
|
||||
app: &AppState,
|
||||
scope: BackupScope,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Data => {
|
||||
admin_state
|
||||
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
warn!(
|
||||
error_category = "system_import_coordination_failed",
|
||||
lock_error = ?error,
|
||||
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||
);
|
||||
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?
|
||||
.map_err(S3BackupTaskError::from)
|
||||
}
|
||||
|
||||
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| {
|
||||
app.encryption_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_s3_backup_config_for_run(
|
||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
.or_else(|| admin_system_config_default_value(key));
|
||||
if let Some(value) = value {
|
||||
let value = if *key == "backup_s3_secret_access_key" {
|
||||
decrypt_s3_secret_access_key(app, value)?
|
||||
decrypt_s3_secret_access_key(app, value).await?
|
||||
} else {
|
||||
value
|
||||
};
|
||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(ciphertext) = value
|
||||
async fn decrypt_s3_secret_access_key(
|
||||
app: &AppState,
|
||||
value: Value,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(stored_value) = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Ok(value);
|
||||
};
|
||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
||||
else {
|
||||
return Err(S3BackupTaskError::bad_request(
|
||||
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||
app,
|
||||
"backup_s3_secret_access_key",
|
||||
stored_value.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
S3BackupTaskError::bad_request(
|
||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||
));
|
||||
};
|
||||
)
|
||||
})?;
|
||||
Ok(Value::String(plaintext))
|
||||
}
|
||||
|
||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||
json!({
|
||||
"scope": result.scope.as_config_value(),
|
||||
"bucket": result.bucket,
|
||||
"object_key": result.object_key,
|
||||
"bytes": result.bytes,
|
||||
"sha256": result.sha256,
|
||||
"export_version": result.export_version,
|
||||
"exported_at": result.exported_at,
|
||||
"compression": result.compression,
|
||||
"deleted_old_objects": result.deleted_old_objects,
|
||||
"encryption": result.encryption,
|
||||
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
@@ -603,9 +801,78 @@ mod tests {
|
||||
};
|
||||
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::handlers::shared::decrypt_system_config_secret;
|
||||
use crate::state::AppState;
|
||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||
|
||||
#[test]
|
||||
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||
Ok(())
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||
Err(super::BackupLockRenewalFailure::Lost)
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||
Err(super::BackupLockRenewalFailure::Backend(
|
||||
"redis unavailable"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||
let backup = async move {
|
||||
std::future::pending::<()>().await;
|
||||
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||
Ok::<(), super::S3BackupTaskError>(())
|
||||
};
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||
let app = AppState::new().expect("app state should build");
|
||||
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the system import lease");
|
||||
|
||||
let error = super::build_s3_backup_payload_exclusively(
|
||||
&app,
|
||||
crate::backup::scopes::BackupScope::Config,
|
||||
)
|
||||
.await
|
||||
.expect_err("backup snapshot must not overlap a system import");
|
||||
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
}
|
||||
|
||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||
vec![
|
||||
(
|
||||
@@ -632,6 +899,92 @@ mod tests {
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||
let plaintext = "legacy-s3-secret-access-key";
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(plaintext);
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let values = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect("legacy S3 config should load");
|
||||
assert_eq!(
|
||||
values.get("backup_s3_secret_access_key"),
|
||||
Some(&serde_json::json!(plaintext))
|
||||
);
|
||||
|
||||
let stored = app
|
||||
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read")
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.expect("stored S3 secret should remain a string");
|
||||
assert_ne!(stored, plaintext);
|
||||
assert_eq!(
|
||||
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||
.expect("migrated S3 secret should decrypt"),
|
||||
plaintext
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||
let plaintext = "s3-secret-from-unavailable-key";
|
||||
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||
.expect("unknown-key fixture should encrypt");
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(ciphertext.clone());
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let error = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||
let error_text = error.to_string();
|
||||
assert!(!error_text.contains(plaintext));
|
||||
assert!(!error_text.contains(&ciphertext));
|
||||
assert_eq!(
|
||||
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read"),
|
||||
Some(serde_json::json!(ciphertext))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||
"postgresql://admin:[email protected]/aether".to_string(),
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
error.status(),
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
assert!(!error.detail().contains("database-secret"));
|
||||
}
|
||||
|
||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||
let now = now_unix_secs();
|
||||
StoredBackgroundTaskRun {
|
||||
@@ -774,7 +1127,9 @@ mod tests {
|
||||
|
||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||
|
||||
assert!(payload["bucket"].is_string());
|
||||
assert!(payload.get("bucket").is_none());
|
||||
assert!(payload.get("prefix").is_none());
|
||||
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||
assert!(!payload.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
||||
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||
warn!(
|
||||
error_category = "schedule_tick_failed",
|
||||
"S3 backup schedule tick failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
||||
) -> Result<(), GatewayError> {
|
||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||
Ok(values) => values,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config load failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_load_failed",
|
||||
"S3 backup schedule config load failed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let config = match S3BackupConfig::from_json_map(&values) {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_invalid",
|
||||
"S3 backup schedule config is invalid"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
||||
|
||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "task_submission_failed",
|
||||
"S3 backup scheduled task submission failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||
system_config_bool, system_config_string,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use serde_json::{json, Value};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||
@@ -10,8 +12,19 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||
|
||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BarkPushConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) device_key: Option<String>,
|
||||
@@ -19,6 +32,21 @@ pub(crate) struct BarkPushConfig {
|
||||
pub(crate) template: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BarkPushConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BarkPushConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field(
|
||||
"device_key",
|
||||
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("server_url", &self.server_url)
|
||||
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||
let value = state
|
||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||
@@ -35,23 +63,34 @@ pub(crate) async fn read_bark_push_config(
|
||||
state: &AppState,
|
||||
) -> Result<BarkPushConfig, GatewayError> {
|
||||
let enabled = bark_push_module_enabled(state).await?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let server_url = state
|
||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let server_url = normalized_bark_server_url(&server_url)?;
|
||||
let binding = bark_device_key_binding(&server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
let device_key = match device_key {
|
||||
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(device_key) = device_key.as_deref() {
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
}
|
||||
let template = state
|
||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
if let Some(template) = template.as_deref() {
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
}
|
||||
|
||||
Ok(BarkPushConfig {
|
||||
enabled,
|
||||
@@ -62,7 +101,7 @@ pub(crate) async fn read_bark_push_config(
|
||||
}
|
||||
|
||||
pub(crate) async fn send_bark_push(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
config: &BarkPushConfig,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
@@ -76,11 +115,13 @@ pub(crate) async fn send_bark_push(
|
||||
"Bark Device Key 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
||||
let response = state
|
||||
.client
|
||||
.post(format!("{server_url}/push"))
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||
let response = client
|
||||
.post(push_url)
|
||||
.json(&json!({
|
||||
"device_key": device_key,
|
||||
"title": title,
|
||||
@@ -88,16 +129,14 @@ pub(crate) async fn send_bark_push(
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||
let status = response.status();
|
||||
let text = response
|
||||
.text()
|
||||
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
if !status.is_success() {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 返回 HTTP {status}: {text}"
|
||||
)));
|
||||
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||
}
|
||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||
let code_is_ok = payload
|
||||
@@ -114,53 +153,260 @@ pub(crate) async fn send_bark_push(
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if !code_is_ok {
|
||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
||||
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
let server_url = server_url.trim().trim_end_matches('/');
|
||||
if server_url.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(server_url.to_string())
|
||||
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
|
||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
||||
match template {
|
||||
Some(template) if !template.trim().is_empty() => template
|
||||
.replace("{title}", title)
|
||||
.replace("{body}", markdown_body),
|
||||
_ => markdown_body.to_string(),
|
||||
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||
match error {
|
||||
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||
format!("Bark 响应超过 {max_bytes} 字节")
|
||||
}
|
||||
aether_http::ResponseBodyReadError::Read(error) => {
|
||||
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else {
|
||||
"transport"
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
canonical_bark_server_url(server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||
}
|
||||
|
||||
async fn build_bark_push_client_and_url(
|
||||
server_url: &str,
|
||||
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let normalized = normalized_bark_server_url(server_url)?;
|
||||
let mut push_url = url::Url::parse(&normalized)
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||
|
||||
let host = push_url
|
||||
.host_str()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||
.to_string();
|
||||
let port = push_url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||
let addresses = aether_http::lookup_host_with_limits(
|
||||
host.as_str(),
|
||||
port,
|
||||
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error.kind() {
|
||||
std::io::ErrorKind::TimedOut => "Bark 服务器 DNS 解析超时",
|
||||
std::io::ErrorKind::InvalidData => "Bark 服务器 DNS 解析返回过多地址",
|
||||
_ => "Bark 服务器 DNS 解析失败",
|
||||
};
|
||||
GatewayError::Internal(message.to_string())
|
||||
})?;
|
||||
let allow_benchmarking_ip = push_url.scheme() == "https"
|
||||
&& push_url.port_or_known_default() == Some(443)
|
||||
&& host.eq_ignore_ascii_case("api.day.app");
|
||||
validate_bark_resolved_addresses(
|
||||
&addresses,
|
||||
env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV),
|
||||
allow_benchmarking_ip,
|
||||
)?;
|
||||
|
||||
push_url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||
.pop_if_empty()
|
||||
.push("push");
|
||||
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||
}
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||
Ok((client, push_url))
|
||||
}
|
||||
|
||||
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||
if url.scheme() == "http" && !allow_http {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_resolved_addresses(
|
||||
addresses: &[SocketAddr],
|
||||
allow_private: bool,
|
||||
allow_benchmarking_ip: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||
));
|
||||
}
|
||||
if !allow_private
|
||||
&& addresses.iter().any(|address| {
|
||||
aether_http::is_private_or_reserved_ip(address.ip())
|
||||
&& !(allow_benchmarking_ip
|
||||
&& aether_http::is_ipv4_benchmarking_fake_ip(address.ip()))
|
||||
})
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn env_flag_enabled(key: &str) -> bool {
|
||||
std::env::var(key).ok().is_some_and(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_bark_config_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_content_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_bark_config_field(field, value, max_bytes)
|
||||
}
|
||||
|
||||
fn render_bark_body(
|
||||
template: Option<&str>,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let template = template
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or("{body}");
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
|
||||
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||
let mut cursor = 0usize;
|
||||
while cursor < template.len() {
|
||||
let remaining = &template[cursor..];
|
||||
let title_match = remaining.find("{title}");
|
||||
let body_match = remaining.find("{body}");
|
||||
let next = match (title_match, body_match) {
|
||||
(None, None) => {
|
||||
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||
cursor = template.len();
|
||||
continue;
|
||||
}
|
||||
(Some(index), None) => (index, "{title}", title),
|
||||
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||
(title_index, "{title}", title)
|
||||
}
|
||||
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||
};
|
||||
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||
cursor += next.0 + next.1.len();
|
||||
}
|
||||
if rendered.is_empty() && template.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
Ok(rendered)
|
||||
}
|
||||
|
||||
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||
let next_len = output
|
||||
.len()
|
||||
.checked_add(part.len())
|
||||
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark rendered body exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
output.push_str(part);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{normalized_bark_server_url, render_bark_body};
|
||||
use super::{
|
||||
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||
};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
#[test]
|
||||
fn bark_body_uses_template_when_provided() {
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||
.expect("template should render");
|
||||
assert_eq!(rendered, "告警\n\n原始正文");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
||||
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -170,4 +416,61 @@ mod tests {
|
||||
"https://api.day.app"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||
for invalid in [
|
||||
"https://[email protected]",
|
||||
"https://example.com?target=internal",
|
||||
"https://example.com/#fragment",
|
||||
] {
|
||||
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_http_transport_requires_explicit_opt_in() {
|
||||
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_private_targets_require_explicit_opt_in() {
|
||||
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&private, false, false).is_err());
|
||||
assert!(validate_bark_resolved_addresses(&private, true, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_builtin_server_allows_benchmarking_ip_only_with_https_default_port() {
|
||||
let fake = [SocketAddr::from(([198, 18, 75, 234], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, true).is_ok());
|
||||
assert!(validate_bark_resolved_addresses(
|
||||
&[fake[0], SocketAddr::from(([127, 0, 0, 1], 443))],
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.is_err());
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, false).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||
let secret = "bark-secret-query";
|
||||
let error = reqwest::Client::new()
|
||||
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||
|
||||
let message = bark_request_error_message(&error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
|
||||
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||
let message = bark_response_body_error_message(&body_error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@
|
||||
#[path = "support/responses_ws_probe.rs"]
|
||||
mod responses_ws_probe;
|
||||
|
||||
use aether_gateway::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
use aether_gateway::{codex_client_originator, codex_client_user_agent};
|
||||
use clap::Parser;
|
||||
use http::header::{AUTHORIZATION, USER_AGENT};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
@@ -78,14 +78,12 @@ fn handshake_headers(access_token: &str, account_id: &str) -> Result<HeaderMap,
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(AUTHORIZATION, bearer_authorization_value(access_token)?);
|
||||
headers.insert(HeaderName::from_static("chatgpt-account-id"), account_id);
|
||||
headers.insert(
|
||||
USER_AGENT,
|
||||
HeaderValue::from_static(CODEX_CLIENT_USER_AGENT),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("originator"),
|
||||
HeaderValue::from_static(CODEX_CLIENT_ORIGINATOR),
|
||||
);
|
||||
let user_agent = HeaderValue::from_str(&codex_client_user_agent())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(USER_AGENT, user_agent);
|
||||
let originator = HeaderValue::from_str(&codex_client_originator())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(HeaderName::from_static("originator"), originator);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -111,6 +109,18 @@ mod tests {
|
||||
assert!(headers.contains_key("chatgpt-account-id"));
|
||||
assert!(headers.contains_key(USER_AGENT));
|
||||
assert!(headers.contains_key("originator"));
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(USER_AGENT)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("originator")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_originator().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
CodexResponsesProbeProfile::sent_header_names(),
|
||||
vec![
|
||||
|
||||
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||
{
|
||||
return Err(ProbeFailure::InvalidEndpoint);
|
||||
}
|
||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
||||
|
||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||
let client = wreq::Client::builder()
|
||||
.no_proxy()
|
||||
.connect_timeout(config.turn_timeout)
|
||||
.timeout(config.turn_timeout)
|
||||
.build()
|
||||
@@ -434,7 +436,14 @@ mod tests {
|
||||
#[test]
|
||||
fn probe_url_rejects_credentials_and_query_strings() {
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||
}
|
||||
|
||||
@@ -422,6 +422,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Vendored
+2
@@ -4,6 +4,7 @@ mod auth_runtime;
|
||||
mod candidate_page;
|
||||
mod dashboard_response;
|
||||
mod direct_plan_bypass;
|
||||
mod overview_total;
|
||||
mod scheduler_affinity;
|
||||
mod system_config;
|
||||
|
||||
@@ -30,6 +31,7 @@ pub(crate) use candidate_page::{
|
||||
};
|
||||
pub(crate) use dashboard_response::DashboardResponseCache;
|
||||
pub(crate) use direct_plan_bypass::DirectPlanBypassCache;
|
||||
pub(crate) use overview_total::{OverviewTotalCache, OverviewTotalRead};
|
||||
pub(crate) use scheduler_affinity::{
|
||||
SchedulerAffinityCache, SchedulerAffinitySnapshotEntry, SchedulerAffinityTarget,
|
||||
};
|
||||
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_data_contracts::repository::usage::StoredUsageDashboardAnalytics;
|
||||
|
||||
const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
const FAILURE_BACKOFF: Duration = Duration::from_secs(10);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct OverviewTotalCache {
|
||||
state: Mutex<CacheState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct CacheState {
|
||||
value: Option<(Instant, Arc<StoredUsageDashboardAnalytics>)>,
|
||||
refreshing: bool,
|
||||
retry_after: Option<Instant>,
|
||||
}
|
||||
|
||||
pub(crate) enum OverviewTotalRead {
|
||||
Pending,
|
||||
Failed,
|
||||
Ready {
|
||||
snapshot: Arc<StoredUsageDashboardAnalytics>,
|
||||
stale: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Owns the single refresh slot even if the request that launched it disconnects.
|
||||
/// Dropping a cancelled or panicking worker also releases the slot with backoff.
|
||||
pub(crate) struct OverviewTotalRefresh {
|
||||
cache: Arc<OverviewTotalCache>,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl OverviewTotalCache {
|
||||
pub(crate) fn read(
|
||||
self: &Arc<Self>,
|
||||
now: Instant,
|
||||
) -> (OverviewTotalRead, Option<OverviewTotalRefresh>) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let fresh = state
|
||||
.value
|
||||
.as_ref()
|
||||
.is_some_and(|(at, _)| now.saturating_duration_since(*at) < FRESH_FOR);
|
||||
let retry_allowed = state.retry_after.is_none_or(|after| now >= after);
|
||||
let refresh = if !fresh && !state.refreshing && retry_allowed {
|
||||
state.refreshing = true;
|
||||
Some(OverviewTotalRefresh {
|
||||
cache: Arc::clone(self),
|
||||
completed: false,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = match &state.value {
|
||||
Some((_, snapshot)) => OverviewTotalRead::Ready {
|
||||
snapshot: Arc::clone(snapshot),
|
||||
stale: !fresh,
|
||||
},
|
||||
None if state.refreshing => OverviewTotalRead::Pending,
|
||||
None => OverviewTotalRead::Failed,
|
||||
};
|
||||
(result, refresh)
|
||||
}
|
||||
}
|
||||
|
||||
impl OverviewTotalRefresh {
|
||||
pub(crate) fn finish(mut self, snapshot: Option<StoredUsageDashboardAnalytics>, now: Instant) {
|
||||
let mut state = self
|
||||
.cache
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
state.refreshing = false;
|
||||
if let Some(snapshot) = snapshot {
|
||||
state.value = Some((now, Arc::new(snapshot)));
|
||||
state.retry_after = None;
|
||||
} else {
|
||||
state.retry_after = Some(now + FAILURE_BACKOFF);
|
||||
}
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OverviewTotalRefresh {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
let mut state = self
|
||||
.cache
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
state.refreshing = false;
|
||||
state.retry_after = Some(Instant::now() + FAILURE_BACKOFF);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn snapshot() -> StoredUsageDashboardAnalytics {
|
||||
let mut snapshot = StoredUsageDashboardAnalytics::default();
|
||||
snapshot.total.generated_at = "2026-09-18T00:00:00Z".into();
|
||||
snapshot.total.read_revision = "revision-1".into();
|
||||
snapshot.total.summary.request_count = 42;
|
||||
snapshot
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_cold_reads_claim_one_refresh() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let barrier = Arc::new(std::sync::Barrier::new(16));
|
||||
let now = Instant::now();
|
||||
let workers = (0..16)
|
||||
.map(|_| {
|
||||
let cache = Arc::clone(&cache);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
std::thread::spawn(move || {
|
||||
barrier.wait();
|
||||
let (read, refresh) = cache.read(now);
|
||||
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||
refresh
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut refreshes = workers
|
||||
.into_iter()
|
||||
.filter_map(|worker| worker.join().unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(refreshes.len(), 1);
|
||||
refreshes.pop().unwrap().finish(Some(snapshot()), now);
|
||||
let (read, refresh) = cache.read(now);
|
||||
assert!(matches!(
|
||||
read,
|
||||
OverviewTotalRead::Ready { stale: false, .. }
|
||||
));
|
||||
assert!(refresh.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiration_returns_original_snapshot_and_failed_refresh_preserves_it() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let now = Instant::now();
|
||||
cache.read(now).1.unwrap().finish(Some(snapshot()), now);
|
||||
assert!(cache
|
||||
.read(now + FRESH_FOR - Duration::from_secs(1))
|
||||
.1
|
||||
.is_none());
|
||||
let expired = now + FRESH_FOR;
|
||||
let (read, refresh) = cache.read(expired);
|
||||
let OverviewTotalRead::Ready {
|
||||
snapshot: old,
|
||||
stale: true,
|
||||
} = read
|
||||
else {
|
||||
panic!("expired success must remain visible")
|
||||
};
|
||||
assert_eq!(old.total.generated_at, "2026-09-18T00:00:00Z");
|
||||
assert_eq!(old.total.read_revision, "revision-1");
|
||||
assert!(cache.read(expired).1.is_none());
|
||||
refresh.unwrap().finish(None, expired);
|
||||
let (read, retry) = cache.read(expired + FAILURE_BACKOFF - Duration::from_secs(1));
|
||||
let OverviewTotalRead::Ready {
|
||||
snapshot: retained,
|
||||
stale: true,
|
||||
} = read
|
||||
else {
|
||||
panic!("failed refresh must retain stale success")
|
||||
};
|
||||
assert!(Arc::ptr_eq(&old, &retained));
|
||||
assert!(retry.is_none());
|
||||
assert!(cache.read(expired + FAILURE_BACKOFF).1.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_failure_and_worker_cancellation_back_off_before_retrying() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let now = Instant::now();
|
||||
cache.read(now).1.unwrap().finish(None, now);
|
||||
let (read, refresh) = cache.read(now + Duration::from_secs(9));
|
||||
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||
assert!(refresh.is_none());
|
||||
let (read, refresh) = cache.read(now + FAILURE_BACKOFF);
|
||||
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||
drop(refresh);
|
||||
let after_cancel = Instant::now();
|
||||
let (read, refresh) = cache.read(after_cancel);
|
||||
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||
assert!(refresh.is_none());
|
||||
assert!(cache.read(after_cancel + FAILURE_BACKOFF).1.is_some());
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate(&self, key: &str) {
|
||||
let Ok(_mutation) = self.mutation.lock() else {
|
||||
return;
|
||||
};
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
self.entries.remove(&key.to_string());
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn insert_if_generation(
|
||||
&self,
|
||||
key: String,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-forwarded-by";
|
||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-owner-instance-id";
|
||||
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||
"local_proxy_passthrough_removed";
|
||||
@@ -125,6 +126,7 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1/messages/count_tokens",
|
||||
"/v1/responses",
|
||||
"/v1/responses/compact",
|
||||
"/v1/memories/trace_summarize",
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/calls",
|
||||
"/v1/live",
|
||||
@@ -139,6 +141,8 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/models/{model}/operations/{id}",
|
||||
"/v1beta/operations",
|
||||
"/v1beta/operations/{id}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{path...}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{path...}",
|
||||
"/upload/v1beta/files",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user