i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 09:50:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/api key concurrency runtime miss (#309)

Browse Source 
* test(cli): 覆盖 API key 并发等待与超时路径

* feat(scheduler): API key 并发饱和时等待可用槽位

* fix(proxy): 区分 API key 并发受限与真正的 runtime miss

* fix(outcome): runtime miss 仅归因真实执行候选

* feat(api-keys): 统一 concurrent_limit 默认值与校验辅助

* feat(admin): 独立 Key 接口支持 concurrent_limit

* feat(admin): 用户 API Key 路由支持 concurrent_limit

* feat(public): 自助 API Key 路由支持 concurrent_limit

* feat(import): 导入与存储层持久化 concurrent_limit

* feat(frontend): 同步 API Key concurrent_limit 类型定义

* feat(frontend): 独立 Key 表单支持 concurrent_limit

* feat(frontend): 管理员用户 API Key 表单支持 concurrent_limit

* feat(frontend): 自助 API Key 页面支持 concurrent_limit

* chore(fmt): 统一 runtime 归因相关 Rust 格式

* chore(fmt): 统一 admin API key 路由 Rust 格式

* chore(fmt): 统一 public 路由与相关测试 Rust 格式

* fix(test): 对齐 no-execution usage 归因断言

* test(middleware): 固定 access log tracing 用例线程模型

* fix(frontend): 提取用户 API Key payload 默认并发辅助

* fix(frontend): 保留用户 Key 的 concurrent_limit 默认值

* fix(api-keys): remove hardcoded concurrent limit default

---------

Co-authored-by: fawney19 <elky0401@gmail.com>
This commit is contained in:
RWDai
2026-04-17 14:21:43 +08:00
committed by GitHub
parent e5d3722adf
commit b8702ae124
39 changed files with 1739 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

736
apps/aether-gateway/src/tests/ai_execute/sync/cli.rs
View File

@@ -3,6 +3,9 @@ use super::{
to_bytes, Arc, Body, Json, Mutex, Request, Router, StatusCode,
EXECUTION_PATH_EXECUTION_RUNTIME_SYNC, EXECUTION_PATH_HEADER, TRACE_ID_HEADER,
};
use crate::constants::{
EXECUTION_PATH_LOCAL_API_KEY_CONCURRENCY_LIMITED, LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER,
};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
@@ -14,7 +17,8 @@ use aether_data_contracts::repository::candidate_selection::{
StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
};
use aether_data_contracts::repository::candidates::{
RequestCandidateReadRepository, RequestCandidateStatus,
RequestCandidateReadRepository, RequestCandidateStatus, RequestCandidateWriteRepository,
UpsertRequestCandidateRecord,
};
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
@@ -535,6 +539,736 @@ async fn gateway_executes_openai_cli_sync_via_local_decision_gate_with_local_syn
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_waits_for_api_key_concurrency_slot_then_executes_openai_cli_sync() {
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:cli"])),
Some(serde_json::json!(["gpt-5"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:cli"])),
Some(serde_json::json!(["gpt-5"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-openai-cli-local-limit-1".to_string(),
provider_name: "openai".to_string(),
provider_type: "custom".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-openai-cli-local-limit-1".to_string(),
endpoint_api_format: "openai:cli".to_string(),
endpoint_api_family: Some("openai".to_string()),
endpoint_kind: Some("cli".to_string()),
endpoint_is_active: true,
key_id: "key-openai-cli-local-limit-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["openai:cli".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(serde_json::json!({"openai:cli": 1})),
model_id: "model-openai-cli-local-limit-1".to_string(),
global_model_id: "global-model-openai-cli-local-limit-1".to_string(),
global_model_name: "gpt-5".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(true),
model_provider_model_name: "gpt-5-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "gpt-5-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["openai:cli".to_string()]),
}]),
model_supports_streaming: Some(true),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-cli-local-limit-1".to_string(),
"openai".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-cli-local-limit-1".to_string(),
"provider-openai-cli-local-limit-1".to_string(),
"openai:cli".to_string(),
Some("openai".to_string()),
Some("cli".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.example/custom/v1/responses".to_string(),
None,
None,
Some(2),
Some("/custom/v1/responses".to_string()),
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-cli-local-limit-1".to_string(),
"provider-openai-cli-local-limit-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["openai:cli"])),
encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
"sk-upstream-openai-cli-limit",
)
.expect("api key should encrypt"),
None,
None,
Some(serde_json::json!({"openai:cli": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let execution_runtime_hits = Arc::new(Mutex::new(0usize));
let execution_runtime_hits_clone = Arc::clone(&execution_runtime_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let now_unix_ms = chrono::Utc::now().timestamp_millis().max(0);
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
aether_data_contracts::repository::candidates::StoredRequestCandidate::new(
"cand-pending-openai-cli-local-limit-1".to_string(),
"req-inflight-openai-cli-local-limit-1".to_string(),
Some("user-openai-cli-local-limit-123".to_string()),
Some("key-openai-cli-local-limit-123".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
0,
0,
Some("provider-openai-cli-local-limit-1".to_string()),
Some("endpoint-openai-cli-local-limit-1".to_string()),
Some("key-openai-cli-local-limit-1".to_string()),
RequestCandidateStatus::Pending,
None,
false,
None,
None,
None,
None,
None,
None,
None,
now_unix_ms,
Some(now_unix_ms),
None,
)
.expect("pending candidate should build"),
]));
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "cli",
"auth_endpoint_signature": "openai:cli",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-cli-local-limit-123",
"api_key_id": "key-openai-cli-local-limit-123",
"access_allowed": true
},
"public_path": "/v1/responses"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(|_request: Request| async move { Json(json!({"action": "proxy_public"})) }),
)
.route(
"/api/internal/gateway/plan-sync",
any(|_request: Request| async move { Json(json!({"action": "proxy_public"})) }),
)
.route(
"/v1/responses",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |_request: Request| {
let execution_runtime_hits_inner = Arc::clone(&execution_runtime_hits_clone);
async move {
*execution_runtime_hits_inner
.lock()
.expect("mutex should lock") += 1;
Json(json!({
"request_id": "trace-openai-cli-local-limit-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "resp-cli-local-limit-123",
"object": "response",
"model": "gpt-5-upstream",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 2,
"total_tokens": 3
}
}
},
"telemetry": {
"elapsed_ms": 21
}
}))
}
}),
);
let mut auth_snapshot = sample_auth_snapshot(
"key-openai-cli-local-limit-123",
"user-openai-cli-local-limit-123",
);
auth_snapshot.api_key_concurrent_limit = Some(1);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-cli-local-limit")),
auth_snapshot,
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let (_upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let request_candidate_repository_for_release = Arc::clone(&request_candidate_repository);
let release_inflight_candidate = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
let pending = request_candidate_repository_for_release
.list_by_request_id("req-inflight-openai-cli-local-limit-1")
.await
.expect("inflight candidates should read")
.into_iter()
.find(|candidate| candidate.id == "cand-pending-openai-cli-local-limit-1")
.expect("seeded inflight candidate should exist");
request_candidate_repository_for_release
.upsert(UpsertRequestCandidateRecord {
id: pending.id,
request_id: pending.request_id,
user_id: pending.user_id,
api_key_id: pending.api_key_id,
username: pending.username,
api_key_name: pending.api_key_name,
candidate_index: pending.candidate_index,
retry_index: pending.retry_index,
provider_id: pending.provider_id,
endpoint_id: pending.endpoint_id,
key_id: pending.key_id,
status: RequestCandidateStatus::Success,
skip_reason: None,
is_cached: Some(false),
status_code: Some(200),
error_type: None,
error_message: None,
latency_ms: Some(1),
concurrent_requests: pending.concurrent_requests,
extra_data: pending.extra_data,
required_capabilities: pending.required_capabilities,
created_at_unix_ms: Some(pending.created_at_unix_ms),
started_at_unix_ms: pending.started_at_unix_ms,
finished_at_unix_ms: Some(pending.created_at_unix_ms.saturating_add(25)),
})
.await
.expect("inflight candidate should update");
});
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/responses"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-cli-local-limit",
)
.header(TRACE_ID_HEADER, "trace-openai-cli-local-limit-123")
.body("{\"model\":\"gpt-5\",\"input\":\"hello\",\"store\":false}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_EXECUTION_RUNTIME_SYNC)
);
assert_eq!(
response
.headers()
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
.and_then(|value| value.to_str().ok()),
None
);
let payload: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(payload["model"], "gpt-5-upstream");
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-cli-local-limit-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
assert_eq!(stored_candidates[0].skip_reason.as_deref(), None);
assert_eq!(
*execution_runtime_hits.lock().expect("mutex should lock"),
1
);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
release_inflight_candidate
.await
.expect("release task should complete");
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_returns_concurrency_limited_after_wait_budget_expires_for_openai_cli_sync() {
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:cli"])),
Some(serde_json::json!(["gpt-5"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:cli"])),
Some(serde_json::json!(["gpt-5"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-openai-cli-local-timeout-1".to_string(),
provider_name: "openai".to_string(),
provider_type: "custom".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-openai-cli-local-timeout-1".to_string(),
endpoint_api_format: "openai:cli".to_string(),
endpoint_api_family: Some("openai".to_string()),
endpoint_kind: Some("cli".to_string()),
endpoint_is_active: true,
key_id: "key-openai-cli-local-timeout-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["openai:cli".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(serde_json::json!({"openai:cli": 1})),
model_id: "model-openai-cli-local-timeout-1".to_string(),
global_model_id: "global-model-openai-cli-local-timeout-1".to_string(),
global_model_name: "gpt-5".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(true),
model_provider_model_name: "gpt-5-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "gpt-5-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["openai:cli".to_string()]),
}]),
model_supports_streaming: Some(true),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-cli-local-timeout-1".to_string(),
"openai".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-cli-local-timeout-1".to_string(),
"provider-openai-cli-local-timeout-1".to_string(),
"openai:cli".to_string(),
Some("openai".to_string()),
Some("cli".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.example/custom/v1/responses".to_string(),
None,
None,
Some(2),
Some("/custom/v1/responses".to_string()),
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-cli-local-timeout-1".to_string(),
"provider-openai-cli-local-timeout-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["openai:cli"])),
encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
"sk-upstream-openai-cli-timeout",
)
.expect("api key should encrypt"),
None,
None,
Some(serde_json::json!({"openai:cli": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let execution_runtime_hits = Arc::new(Mutex::new(0usize));
let execution_runtime_hits_clone = Arc::clone(&execution_runtime_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let now_unix_ms = chrono::Utc::now().timestamp_millis().max(0);
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
aether_data_contracts::repository::candidates::StoredRequestCandidate::new(
"cand-pending-openai-cli-local-timeout-1".to_string(),
"req-inflight-openai-cli-local-timeout-1".to_string(),
Some("user-openai-cli-local-timeout-123".to_string()),
Some("key-openai-cli-local-timeout-123".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
0,
0,
Some("provider-openai-cli-local-timeout-1".to_string()),
Some("endpoint-openai-cli-local-timeout-1".to_string()),
Some("key-openai-cli-local-timeout-1".to_string()),
RequestCandidateStatus::Pending,
None,
false,
None,
None,
None,
None,
None,
None,
None,
now_unix_ms,
Some(now_unix_ms),
None,
)
.expect("pending candidate should build"),
]));
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "cli",
"auth_endpoint_signature": "openai:cli",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-cli-local-timeout-123",
"api_key_id": "key-openai-cli-local-timeout-123",
"access_allowed": true
},
"public_path": "/v1/responses"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(|_request: Request| async move { Json(json!({"action": "proxy_public"})) }),
)
.route(
"/api/internal/gateway/plan-sync",
any(|_request: Request| async move { Json(json!({"action": "proxy_public"})) }),
)
.route(
"/v1/responses",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |_request: Request| {
let execution_runtime_hits_inner = Arc::clone(&execution_runtime_hits_clone);
async move {
*execution_runtime_hits_inner
.lock()
.expect("mutex should lock") += 1;
Json(json!({
"request_id": "trace-openai-cli-local-timeout-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "resp-cli-local-timeout-123",
"object": "response",
"model": "gpt-5-upstream",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 2,
"total_tokens": 3
}
}
},
"telemetry": {
"elapsed_ms": 21
}
}))
}
}),
);
let mut auth_snapshot = sample_auth_snapshot(
"key-openai-cli-local-timeout-123",
"user-openai-cli-local-timeout-123",
);
auth_snapshot.api_key_concurrent_limit = Some(1);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-cli-local-timeout")),
auth_snapshot,
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let (_upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let started_at = std::time::Instant::now();
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/responses"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-cli-local-timeout",
)
.header(TRACE_ID_HEADER, "trace-openai-cli-local-timeout-123")
.body("{\"model\":\"gpt-5\",\"input\":\"hello\",\"store\":false}")
.send()
.await
.expect("request should complete");
assert!(
started_at.elapsed() >= std::time::Duration::from_millis(100),
"request should wait for the bounded concurrency window before failing"
);
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_API_KEY_CONCURRENCY_LIMITED)
);
assert_eq!(
response
.headers()
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
.and_then(|value| value.to_str().ok()),
Some("api_key_concurrency_limit_reached")
);
let payload: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(
payload["error"]["message"],
serde_json::Value::String("当前 API Key 并发请求数已达上限，请稍后重试".to_string())
);
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-cli-local-timeout-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
assert_eq!(
stored_candidates[0].skip_reason.as_deref(),
Some("api_key_concurrency_limit_reached")
);
assert_eq!(
*execution_runtime_hits.lock().expect("mutex should lock"),
0
);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_returns_openai_cli_error_for_local_sync_failure() {
fn hash_api_key(value: &str) -> String {

3
apps/aether-gateway/src/tests/control/admin/api_keys.rs
View File

@@ -447,6 +447,7 @@ async fn gateway_handles_admin_api_keys_create_locally_with_trusted_admin_princi
assert_eq!(payload["name"], json!("standalone-key"));
assert_eq!(payload["is_standalone"], json!(true));
assert_eq!(payload["rate_limit"], serde_json::Value::Null);
assert_eq!(payload["concurrent_limit"], serde_json::Value::Null);
assert_eq!(payload["allowed_providers"], json!(["openai"]));
assert_eq!(payload["allowed_api_formats"], json!(["openai:chat"]));
assert_eq!(payload["allowed_models"], json!(["gpt-4.1"]));
@@ -515,6 +516,7 @@ async fn gateway_handles_admin_api_keys_update_locally_with_trusted_admin_princi
.json(&json!({
"name": "renamed-key",
"rate_limit": null,
"concurrent_limit": 12,
"allowed_providers": ["gemini"],
"allowed_api_formats": ["gemini:chat"],
"allowed_models": ["gemini-2.5-pro"],
@@ -531,6 +533,7 @@ async fn gateway_handles_admin_api_keys_update_locally_with_trusted_admin_princi
assert_eq!(payload["id"], json!("key-123"));
assert_eq!(payload["name"], json!("renamed-key"));
assert_eq!(payload["rate_limit"], serde_json::Value::Null);
assert_eq!(payload["concurrent_limit"], json!(12));
assert_eq!(payload["allowed_providers"], json!(["gemini"]));
assert_eq!(payload["allowed_api_formats"], json!(["gemini:chat"]));
assert_eq!(payload["allowed_models"], json!(["gemini-2.5-pro"]));

3
apps/aether-gateway/src/tests/control/admin/users.rs
View File

@@ -629,6 +629,7 @@ async fn gateway_handles_admin_user_api_key_routes_locally_with_trusted_admin_pr
.expect("json body should parse");
assert_eq!(create_payload["name"], "new-key");
assert_eq!(create_payload["rate_limit"], 90);
assert_eq!(create_payload["concurrent_limit"], serde_json::Value::Null);
assert_eq!(
create_payload["message"],
"API Key创建成功，请妥善保存完整密钥"
@@ -651,6 +652,7 @@ async fn gateway_handles_admin_user_api_key_routes_locally_with_trusted_admin_pr
.json(&json!({
"name": "renamed",
"rate_limit": 120,
"concurrent_limit": 9,
}))
.send()
.await
@@ -664,6 +666,7 @@ async fn gateway_handles_admin_user_api_key_routes_locally_with_trusted_admin_pr
assert_eq!(update_payload["name"], "renamed");
assert_eq!(update_payload["is_locked"], false);
assert_eq!(update_payload["rate_limit"], 120);
assert_eq!(update_payload["concurrent_limit"], 9);
assert_eq!(update_payload["message"], "API Key更新成功");
let lock_response = client

6
apps/aether-gateway/src/tests/frontdoor/public_support.rs
View File

@@ -6206,6 +6206,7 @@ async fn gateway_handles_users_me_api_key_writes_locally_without_proxying_upstre
.to_string();
assert_eq!(create_payload["name"], "writer-key");
assert_eq!(create_payload["rate_limit"], 120);
assert_eq!(create_payload["concurrent_limit"], serde_json::Value::Null);
assert_eq!(create_payload["message"], "API密钥创建成功");
assert!(create_payload["key"]
.as_str()
@@ -6219,7 +6220,8 @@ async fn gateway_handles_users_me_api_key_writes_locally_without_proxying_upstre
.header("user-agent", "AetherTest/1.0")
.json(&json!({
"name": "writer-key-renamed",
"rate_limit": 30
"rate_limit": 30,
"concurrent_limit": 4
}))
.send()
.await
@@ -6231,6 +6233,7 @@ async fn gateway_handles_users_me_api_key_writes_locally_without_proxying_upstre
.expect("json body should parse");
assert_eq!(update_payload["name"], "writer-key-renamed");
assert_eq!(update_payload["rate_limit"], 30);
assert_eq!(update_payload["concurrent_limit"], 4);
assert_eq!(update_payload["message"], "API密钥已更新");
let toggle_response = client
@@ -6308,6 +6311,7 @@ async fn gateway_handles_users_me_api_key_writes_locally_without_proxying_upstre
detail_payload["allowed_providers"],
json!(["provider-openai"])
);
assert_eq!(detail_payload["concurrent_limit"], 4);
assert_eq!(detail_payload["force_capabilities"], json!({}));
let delete_response = client

11
apps/aether-gateway/src/tests/usage/local.rs
View File

@@ -1333,14 +1333,14 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
stored_usage.user_id.as_deref(),
Some("user-claude-cli-usage-local-miss-1")
);
assert_eq!(stored_usage.provider_name, "RightCode");
assert_eq!(stored_usage.provider_name, "claude");
assert_eq!(stored_usage.model, "gpt-5.4");
assert_eq!(stored_usage.api_format.as_deref(), Some("claude:cli"));
assert_eq!(
stored_usage.endpoint_api_format.as_deref(),
Some("openai:cli")
Some("claude:cli")
);
assert_eq!(stored_usage.routing_key_name(), Some("codex"));
assert_eq!(stored_usage.routing_key_name(), None);
assert_eq!(stored_usage.routing_planner_kind(), Some("claude_cli_sync"));
assert_eq!(stored_usage.routing_route_family(), Some("claude"));
assert_eq!(stored_usage.routing_route_kind(), Some("cli"));
@@ -1375,10 +1375,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
stored_candidates[0].skip_reason.as_deref(),
Some("format_conversion_disabled")
);
assert_eq!(
stored_usage.routing_candidate_id(),
Some(stored_candidates[0].id.as_str())
);
assert_eq!(stored_usage.routing_candidate_id(), None);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();