Merge upstream/main into feat/one-click-update

This commit is contained in:
zhiqicloud
2026-05-22 15:28:27 +08:00
269 changed files with 31793 additions and 6990 deletions

View File

@@ -85,12 +85,12 @@ ADMIN_USERNAME=admin123456
# MAX = CPU 核数 × 10AI 网关偏 IO 等待,可激进些;纯 OLTP 用 × 4
# MIN = MAX × 0.2(保留常驻连接应对突发流量,避免冷启动握手开销)
# 多实例部署时请按 实例数 × MAX 控制总和PG 端 max_connections 至少为该总和 + 20 余量
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=1
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=4
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=20
# AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_CACHE_CAPACITY=100
# AETHER_GATEWAY_DATA_POSTGRES_ACQUIRE_TIMEOUT_MS=3000
# PostgreSQL 性能调优(默认值适合 2核2GB 的最小机器,按实际配置覆盖)
# PostgreSQL 性能调优(默认值适合 2核4GB 机器,按实际配置覆盖)
# 参考shared_buffers ≈ 可用内存 25%effective_cache_size ≈ 可用内存 50-75%
# work_mem 是每个连接每个排序操作的内存,不要设太大(并发数 × work_mem 是实际占用)
# | 系统内存 | shared_buffers | effective_cache_size | work_mem |
@@ -99,7 +99,7 @@ ADMIN_USERNAME=admin123456
# | 8GB | 2GB | 6GB | 16MB |
# | 16GB | 4GB | 12GB | 32MB |
# | 32GB+ | 8GB | 24GB | 32MB |
# POSTGRES_SHARED_BUFFERS=256MB
# POSTGRES_EFFECTIVE_CACHE_SIZE=768MB
# POSTGRES_WORK_MEM=4MB
# POSTGRES_MAINTENANCE_WORK_MEM=64MB
# POSTGRES_SHARED_BUFFERS=1GB
# POSTGRES_EFFECTIVE_CACHE_SIZE=3GB
# POSTGRES_WORK_MEM=16MB
# POSTGRES_MAINTENANCE_WORK_MEM=256MB

17
Cargo.lock generated
View File

@@ -859,6 +859,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "backon"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
dependencies = [
"fastrand",
]
[[package]]
name = "base64"
version = "0.21.7"
@@ -1638,6 +1647,12 @@ dependencies = [
"regex",
]
[[package]]
name = "fastrand"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "filedescriptor"
version = "0.8.3"
@@ -3390,8 +3405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e37ec3fd44bea2ec947ba6cc7634d7999a6590aca7c35827c250bc0de502bda6"
dependencies = [
"arc-swap",
"backon",
"bytes",
"combine",
"futures-channel",
"futures-util",
"itoa",
"num-bigint",

View File

@@ -82,7 +82,7 @@ hmac = "0.12"
http = "1"
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams"] }
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
regex = "1"
rustls = { version = "0.23", features = ["ring"] }
semver = "1"

View File

@@ -335,6 +335,11 @@ if ! command -v curl >/dev/null 2>&1; then
exit 1
fi
if [ -z "$${RUSTC_WRAPPER:-}" ] && command -v sccache >/dev/null 2>&1; then
export RUSTC_WRAPPER="$$(command -v sccache)"
echo "=> 启用 Rust 编译缓存: $${RUSTC_WRAPPER}"
fi
if ! ensure_dev_infra; then
exit 1
fi

View File

@@ -7,10 +7,11 @@ pub(crate) use crate::handlers::admin::{
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
reconcile_admin_fixed_provider_template_endpoints,
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
update_existing_provider_oauth_catalog_key, AdminAppState,
AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError, AdminRequestContext,
AdminRouteRequest, AdminRouteResponse, AdminRouteResult, AdminStatsTimeRange,
AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
OAUTH_REQUEST_FAILED_PREFIX,
};
use crate::handlers::admin::{

View File

@@ -44,11 +44,11 @@ pub(crate) use aether_ai_formats::api::{
build_core_error_body_for_client_format, convert_standard_chat_response,
core_error_background_report_kind, core_error_default_client_api_format,
core_success_background_report_kind, encode_kiro_sse_events,
implicit_sync_finalize_report_kind, is_core_error_finalize_kind,
normalize_provider_private_report_context, normalize_provider_private_response_value,
provider_private_response_allows_sync_finalize, resolve_claude_stream_spec,
resolve_claude_sync_spec, resolve_gemini_stream_spec, resolve_gemini_sync_spec,
resolve_local_image_stream_spec, resolve_local_image_sync_spec,
extract_provider_private_stream_error_body, implicit_sync_finalize_report_kind,
is_core_error_finalize_kind, normalize_provider_private_report_context,
normalize_provider_private_response_value, provider_private_response_allows_sync_finalize,
resolve_claude_stream_spec, resolve_claude_sync_spec, resolve_gemini_stream_spec,
resolve_gemini_sync_spec, resolve_local_image_stream_spec, resolve_local_image_sync_spec,
resolve_local_same_format_stream_spec, resolve_local_same_format_sync_spec,
resolve_openai_embedding_sync_spec, sanitize_request_path_and_query, AiControlPlanRequest,
CanonicalContentPart, CanonicalStreamEvent, CanonicalStreamFrame, ClaudeClientEmitter,

View File

@@ -19,9 +19,10 @@ use crate::ai_serving::{
apply_codex_openai_responses_special_body_edits, apply_codex_openai_responses_special_headers,
build_chatgpt_web_image_request_body,
build_gemini_image_request_body_from_openai_image_request,
build_openai_image_provider_request_body, default_model_for_openai_image_operation,
normalize_openai_image_request, request_conversion_direct_auth, CandidateFailureDiagnostic,
GatewayProviderTransportSnapshot, PlannerAppState, RequestConversionKind,
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
default_model_for_openai_image_operation, normalize_openai_image_request,
request_conversion_direct_auth, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
PlannerAppState, RequestConversionKind,
};
use crate::image_capabilities::openai_image_normalize_options_for_provider;
use crate::AppState;
@@ -160,21 +161,31 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
.provider_type
.trim()
.eq_ignore_ascii_case("grok");
let is_codex = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("codex");
let transport_profile = crate::ai_serving::transport::resolve_transport_profile(transport);
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else if is_grok {
build_grok_upstream_url(transport, GROK_CHAT_PATH)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
build_openai_image_upstream_url(transport, Some(parts.uri.path()), parts.uri.query())
};
let mut provider_request_body = if is_chatgpt_web {
match build_chatgpt_web_image_request_body(parts, body_json, body_base64) {
Ok(body) => body,
Err(err) => err.to_error_json(),
}
} else {
} else if is_codex || is_grok {
build_openai_image_provider_request_body(&normalized_request)
} else {
build_openai_image_api_provider_request_body(
&normalized_request,
Some(prepared_candidate.mapped_model.as_str()),
)
};
if !is_chatgpt_web {
apply_codex_openai_responses_special_body_edits(

View File

@@ -85,7 +85,7 @@ fn injects_stable_prompt_cache_key_for_codex_requests() {
assert_eq!(
body["prompt_cache_key"],
"172c39e6-c0a0-5a70-8b63-e0f8e0d185a3"
"53363264-dbb0-5f9d-b9c7-3e92c45c5bdf"
);
}

View File

@@ -24,9 +24,13 @@ use crate::ai_serving::transport::kiro::{
use crate::ai_serving::transport::{
build_grok_browser_headers, build_grok_upstream_url, build_kiro_cross_format_upstream_url,
build_openai_image_headers, build_openai_image_upstream_url,
build_standard_provider_request_headers, openai_image_transport_unsupported_reason,
resolve_grok_session_auth, resolve_openai_image_auth, GrokHeaderInput,
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
build_standard_provider_request_headers, build_windsurf_cascade_headers,
build_windsurf_cascade_request_body, build_windsurf_cascade_upstream_url,
is_windsurf_provider_transport,
local_windsurf_request_transport_unsupported_reason_with_network,
openai_image_transport_unsupported_reason, resolve_grok_session_auth,
resolve_openai_image_auth, GrokHeaderInput, ProviderOpenAiImageHeadersInput,
StandardProviderRequestHeadersInput, GROK_CHAT_PATH, WINDSURF_ENVELOPE_NAME,
};
use crate::ai_serving::{
build_openai_image_request_body_from_gemini_image_request, gemini_request_is_image_generation,
@@ -61,6 +65,134 @@ fn is_grok_text_provider_api_format(provider_api_format: &str) -> bool {
)
}
fn provider_preserves_claude_thinking_signatures(provider_type: &str, base_url: &str) -> bool {
let provider_type = provider_type.trim().to_ascii_lowercase();
let base_url = base_url.trim().to_ascii_lowercase();
let is_bedrock_runtime_url = base_url.contains("bedrock-runtime")
&& (base_url.contains("amazonaws.com")
|| base_url.contains("amazonaws.com.cn")
|| base_url.contains("api.aws"));
matches!(
provider_type.as_str(),
"anthropic" | "claude_code" | "bedrock" | "aws_bedrock" | "amazon_bedrock"
) || base_url.contains("api.anthropic.com")
|| is_bedrock_runtime_url
}
fn sanitize_claude_thinking_block(block: Value) -> (Option<Value>, bool) {
let Some(object) = block.as_object() else {
return (Some(block), false);
};
let block_type = object
.get("type")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
match block_type {
"thinking" => {
let thinking_text = object
.get("thinking")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if thinking_text.is_empty() {
(None, true)
} else {
(
Some(serde_json::json!({
"type": "text",
"text": thinking_text,
})),
true,
)
}
}
"redacted_thinking" => (None, true),
_ => (Some(block), false),
}
}
fn sanitize_claude_message_content_for_non_native_thinking(content: &mut Value) -> bool {
const OMITTED_THINKING_TEXT: &str = "Previous thinking omitted.";
if content.is_object() {
let original = std::mem::take(content);
let (sanitized, changed) = sanitize_claude_thinking_block(original);
if changed {
*content = sanitized.unwrap_or_else(|| {
serde_json::json!({
"type": "text",
"text": OMITTED_THINKING_TEXT,
})
});
}
return changed;
}
let Some(blocks) = content.as_array_mut() else {
return false;
};
let original_blocks = std::mem::take(blocks);
let mut changed = false;
let mut sanitized_blocks = Vec::with_capacity(original_blocks.len());
for block in original_blocks {
let (sanitized, block_changed) = sanitize_claude_thinking_block(block);
changed |= block_changed;
if let Some(sanitized) = sanitized {
sanitized_blocks.push(sanitized);
}
}
if changed && sanitized_blocks.is_empty() {
sanitized_blocks.push(serde_json::json!({
"type": "text",
"text": OMITTED_THINKING_TEXT,
}));
}
*blocks = sanitized_blocks;
changed
}
fn sanitize_claude_request_thinking_signatures_for_non_native(body_json: &mut Value) -> bool {
body_json
.get_mut("messages")
.and_then(Value::as_array_mut)
.map(|messages| {
messages.iter_mut().fold(false, |changed, message| {
let is_assistant = message
.get("role")
.and_then(Value::as_str)
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"));
if !is_assistant {
return changed;
}
let content_changed = message
.get_mut("content")
.is_some_and(sanitize_claude_message_content_for_non_native_thinking);
changed || content_changed
})
})
.unwrap_or(false)
}
fn apply_non_native_claude_thinking_signature_compat(
provider_request_body: &mut Value,
provider_api_format: &str,
transport: &GatewayProviderTransportSnapshot,
) {
if crate::ai_serving::normalize_api_format_alias(provider_api_format) != "claude:messages" {
return;
}
if provider_preserves_claude_thinking_signatures(
transport.provider.provider_type.as_str(),
transport.endpoint.base_url.as_str(),
) {
return;
}
let _ = sanitize_claude_request_thinking_signatures_for_non_native(provider_request_body);
}
pub(crate) async fn resolve_local_standard_candidate_payload_parts(
state: &AppState,
parts: &http::request::Parts,
@@ -198,11 +330,18 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
return None;
}
if let Some(skip_reason) = crate::ai_serving::request_pair_transport_unsupported_reason(
transport,
spec_metadata.api_format,
provider_api_format,
) {
let is_windsurf_cascade =
provider_api_format == "openai:chat" && is_windsurf_provider_transport(transport);
let transport_unsupported_reason = if is_windsurf_cascade {
local_windsurf_request_transport_unsupported_reason_with_network(transport)
} else {
crate::ai_serving::request_pair_transport_unsupported_reason(
transport,
spec_metadata.api_format,
provider_api_format,
)
};
if let Some(skip_reason) = transport_unsupported_reason {
mark_skipped_local_standard_candidate(
state,
input,
@@ -321,7 +460,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
provider_api_format,
parts.uri.path(),
upstream_is_stream,
if is_kiro_claude_cli {
if is_kiro_claude_cli || is_windsurf_cascade {
None
} else {
transport.endpoint.body_rules.as_ref()
@@ -379,6 +518,11 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
.await;
return None;
}
apply_non_native_claude_thinking_signature_compat(
&mut provider_request_body,
provider_api_format,
transport,
);
if let Some(mapping) =
crate::system_features::reasoning_model_directive_mapping_for_api_format_and_model(
state,
@@ -422,6 +566,11 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
.await;
return None;
}
apply_non_native_claude_thinking_signature_compat(
&mut provider_request_body,
provider_api_format,
transport,
);
}
if let Some(kiro_auth) = kiro_auth.as_ref() {
@@ -443,6 +592,24 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
)
.await;
}
if is_windsurf_cascade {
return build_windsurf_cross_format_payload_parts(
state,
parts,
trace_id,
body_json,
input,
attempt,
transport,
provider_api_format,
prepared_candidate.mapped_model,
prepared_candidate.auth_header,
prepared_candidate.auth_value,
provider_request_body,
upstream_is_stream,
)
.await;
}
let upstream_url = match crate::ai_serving::planner::standard::build_standard_upstream_url(
parts,
@@ -542,6 +709,121 @@ fn apply_transport_request_body_semantics(
)
}
#[allow(clippy::too_many_arguments)]
async fn build_windsurf_cross_format_payload_parts(
state: &AppState,
parts: &http::request::Parts,
trace_id: &str,
original_body_json: &serde_json::Value,
input: &LocalStandardDecisionInput,
attempt: &LocalStandardCandidateAttempt,
transport: &Arc<GatewayProviderTransportSnapshot>,
provider_api_format: &str,
mapped_model: String,
auth_header: String,
auth_value: String,
openai_chat_request_body: Value,
upstream_is_stream: bool,
) -> Option<LocalStandardCandidatePayloadParts> {
let candidate = &attempt.eligible.candidate;
let effective_headers = input.effective_headers(&parts.headers);
let provider_request_body = match build_windsurf_cascade_request_body(
&openai_chat_request_body,
&mapped_model,
&auth_value,
transport.endpoint.body_rules.as_ref(),
Some(effective_headers),
upstream_is_stream,
) {
Some(body) => body,
None => {
mark_skipped_local_standard_candidate_with_extra_data(
state,
input,
trace_id,
candidate,
attempt.candidate_index,
&attempt.candidate_id,
"provider_request_body_build_failed",
request_body_build_failure_extra_data(
&openai_chat_request_body,
provider_api_format,
provider_api_format,
),
)
.await;
return None;
}
};
let upstream_url = match build_windsurf_cascade_upstream_url(
transport.endpoint.base_url.as_str(),
parts.uri.query(),
) {
Some(url) => url,
None => {
mark_skipped_local_standard_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
attempt.candidate_index,
&attempt.candidate_id,
"upstream_url_missing",
CandidateFailureDiagnostic::upstream_url_missing(
provider_api_format,
provider_api_format,
"standard_family_windsurf_url",
),
)
.await;
return None;
}
};
let provider_request_headers = match build_windsurf_cascade_headers(
effective_headers,
&provider_request_body,
original_body_json,
transport.endpoint.header_rules.as_ref(),
&auth_header,
&auth_value,
upstream_is_stream,
) {
Some(headers) => headers,
None => {
mark_skipped_local_standard_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
attempt.candidate_index,
&attempt.candidate_id,
"transport_header_rules_apply_failed",
CandidateFailureDiagnostic::header_rules_apply_failed(
provider_api_format,
provider_api_format,
"standard_family_windsurf_headers",
),
)
.await;
return None;
}
};
Some(LocalStandardCandidatePayloadParts {
auth_header,
auth_value,
mapped_model,
provider_api_format: provider_api_format.to_string(),
provider_request_body,
provider_request_headers,
upstream_url,
upstream_is_stream,
envelope_name: Some(WINDSURF_ENVELOPE_NAME),
transport: Arc::clone(transport),
transport_profile: None,
})
}
async fn resolve_local_gemini_image_to_openai_image_candidate_payload_parts(
state: &AppState,
parts: &http::request::Parts,
@@ -625,7 +907,8 @@ async fn resolve_local_gemini_image_to_openai_image_candidate_payload_parts(
};
let upstream_is_stream = true;
let upstream_url = build_openai_image_upstream_url(transport, None);
let upstream_url =
build_openai_image_upstream_url(transport, Some("/v1/images/generations"), None);
let effective_headers = input.effective_headers(&parts.headers);
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {
@@ -798,3 +1081,95 @@ async fn build_kiro_cross_format_payload_parts(
transport_profile: None,
})
}
#[cfg(test)]
mod tests {
use super::{
provider_preserves_claude_thinking_signatures,
sanitize_claude_request_thinking_signatures_for_non_native,
};
use serde_json::json;
#[test]
fn sanitizes_historical_claude_thinking_for_non_native_relays() {
let mut body = json!({
"model": "claude-opus-4-1",
"messages": [{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "I should keep this short.",
"signature": "sig_123"
},
{
"type": "redacted_thinking",
"data": "opaque"
},
{
"type": "text",
"text": "Done."
}
]
}]
});
assert!(sanitize_claude_request_thinking_signatures_for_non_native(
&mut body
));
assert_eq!(body["messages"][0]["content"][0]["type"], json!("text"));
assert_eq!(
body["messages"][0]["content"][0]["text"],
json!("I should keep this short.")
);
assert_eq!(body["messages"][0]["content"].as_array().unwrap().len(), 2);
assert_eq!(body["messages"][0]["content"][1]["text"], json!("Done."));
}
#[test]
fn inserts_placeholder_when_only_redacted_thinking_would_remain() {
let mut body = json!({
"model": "claude-opus-4-1",
"messages": [{
"role": "assistant",
"content": [{
"type": "redacted_thinking",
"data": "opaque"
}]
}]
});
assert!(sanitize_claude_request_thinking_signatures_for_non_native(
&mut body
));
assert_eq!(body["messages"][0]["content"][0]["type"], json!("text"));
assert_eq!(
body["messages"][0]["content"][0]["text"],
json!("Previous thinking omitted.")
);
}
#[test]
fn official_claude_providers_preserve_thinking_signatures() {
assert!(provider_preserves_claude_thinking_signatures(
"anthropic",
"https://relay.example.com"
));
assert!(provider_preserves_claude_thinking_signatures(
"custom",
"https://api.anthropic.com"
));
assert!(provider_preserves_claude_thinking_signatures(
"aws",
"https://bedrock-runtime.us-east-1.amazonaws.com"
));
assert!(provider_preserves_claude_thinking_signatures(
"amazon_bedrock",
"https://relay.example.com"
));
assert!(!provider_preserves_claude_thinking_signatures(
"openai",
"https://relay.example.com"
));
}
}

View File

@@ -209,7 +209,7 @@ fn local_openai_responses_compact_wrapper_strips_include_for_codex_requests() {
assert_eq!(provider_request_body["instructions"], "");
assert_eq!(
provider_request_body["prompt_cache_key"],
"172c39e6-c0a0-5a70-8b63-e0f8e0d185a3"
"3d2e2842-74cb-55dd-803a-b8940b3500c2"
);
}
@@ -355,7 +355,7 @@ fn injects_codex_prompt_cache_key_for_openai_responses_cross_format_requests() {
assert_eq!(
provider_request_body["prompt_cache_key"],
"172c39e6-c0a0-5a70-8b63-e0f8e0d185a3"
"b4dfeb75-b105-544c-a706-39b92f0bddb0"
);
}
@@ -385,6 +385,6 @@ fn injects_codex_prompt_cache_key_for_openai_chat_cross_format_requests() {
assert_eq!(
provider_request_body["prompt_cache_key"],
"172c39e6-c0a0-5a70-8b63-e0f8e0d185a3"
"4ee6ea6e-3ac6-5a18-8cb8-1f8b956419e5"
);
}

View File

@@ -9,7 +9,6 @@ pub(super) use self::payload::maybe_build_local_openai_chat_decision_payload_for
pub(super) use self::support::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
materialize_local_openai_chat_candidate_attempts, LocalOpenAiChatCandidateAttempt,
LocalOpenAiChatCandidateAttemptSource, LocalOpenAiChatDecisionInput,
};

View File

@@ -28,6 +28,12 @@ use crate::ai_serving::transport::kiro::{
KIRO_ENVELOPE_NAME,
};
use crate::ai_serving::transport::local_openai_chat_transport_unsupported_reason;
use crate::ai_serving::transport::windsurf::{
build_windsurf_cascade_headers, build_windsurf_cascade_request_body,
build_windsurf_cascade_upstream_url, is_windsurf_provider_transport,
local_windsurf_request_transport_unsupported_reason_with_network,
resolve_windsurf_cascade_auth, WINDSURF_ENVELOPE_NAME,
};
use crate::ai_serving::transport::{
build_grok_browser_headers, build_grok_upstream_url, build_kiro_cross_format_upstream_url,
build_openai_image_headers, build_openai_image_upstream_url,
@@ -334,6 +340,25 @@ pub(crate) async fn resolve_local_openai_chat_candidate_payload_parts(
}));
}
if provider_api_format == "openai:chat" && is_windsurf_provider_transport(transport) {
return build_windsurf_openai_chat_payload_parts(
state,
parts,
trace_id,
body_json,
input,
eligible,
candidate_index,
candidate_id,
decision_kind,
report_kind,
transport,
upstream_is_stream,
redaction.redacted,
)
.await;
}
if provider_api_format == "openai:chat" {
if let Some(skip_reason) = local_openai_chat_transport_unsupported_reason(transport) {
mark_skipped_local_openai_chat_candidate(
@@ -909,7 +934,11 @@ async fn resolve_openai_chat_to_openai_image_payload_parts(
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
build_openai_image_upstream_url(
transport,
Some("/v1/images/generations"),
parts.uri.query(),
)
};
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {
@@ -1249,6 +1278,181 @@ fn chatgpt_web_image_internal_url(base_url: &str) -> String {
format!("{base_url}/__aether/chatgpt-web-image")
}
#[allow(clippy::too_many_arguments)]
async fn build_windsurf_openai_chat_payload_parts(
state: &AppState,
parts: &http::request::Parts,
trace_id: &str,
body_json: &serde_json::Value,
input: &LocalOpenAiChatDecisionInput,
eligible: &EligibleLocalExecutionCandidate,
candidate_index: u32,
candidate_id: &str,
decision_kind: &str,
report_kind: &str,
transport: &Arc<GatewayProviderTransportSnapshot>,
upstream_is_stream: bool,
request_redacted: bool,
) -> Result<Option<LocalOpenAiChatCandidatePayloadParts>, GatewayError> {
let planner_state = crate::ai_serving::PlannerAppState::new(state);
let candidate = &eligible.candidate;
if let Some(skip_reason) =
local_windsurf_request_transport_unsupported_reason_with_network(transport)
{
mark_skipped_local_openai_chat_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return Ok(None);
}
let prepared_candidate = match prepare_header_authenticated_candidate(
planner_state,
transport,
candidate,
resolve_windsurf_cascade_auth(transport)
.or_else(|| resolve_local_openai_bearer_auth(transport)),
OauthPreparationContext {
trace_id,
api_format: "openai:chat",
operation: "openai_chat_windsurf_cascade",
},
)
.await
{
Ok(prepared) => prepared,
Err(skip_reason) => {
mark_skipped_local_openai_chat_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return Ok(None);
}
};
let Some(provider_request_body) = build_windsurf_cascade_request_body(
body_json,
&prepared_candidate.mapped_model,
&prepared_candidate.auth_value,
transport.endpoint.body_rules.as_ref(),
Some(&parts.headers),
upstream_is_stream,
) else {
mark_skipped_local_openai_chat_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"provider_request_body_build_failed",
CandidateFailureDiagnostic::envelope_build_failed(
"openai:chat",
"openai:chat",
"openai_chat_windsurf_cascade",
),
)
.await;
return Ok(None);
};
let Some(upstream_url) = build_windsurf_cascade_upstream_url(
transport.endpoint.base_url.as_str(),
parts.uri.query(),
) else {
mark_skipped_local_openai_chat_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"upstream_url_missing",
CandidateFailureDiagnostic::upstream_url_missing(
"openai:chat",
"openai:chat",
"openai_chat_windsurf_url",
),
)
.await;
return Ok(None);
};
let mut provider_request_headers = match build_windsurf_cascade_headers(
&parts.headers,
&provider_request_body,
body_json,
transport.endpoint.header_rules.as_ref(),
&prepared_candidate.auth_header,
&prepared_candidate.auth_value,
upstream_is_stream,
) {
Some(headers) => headers,
None => {
mark_skipped_local_openai_chat_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"transport_header_rules_apply_failed",
CandidateFailureDiagnostic::header_rules_apply_failed(
"openai:chat",
"openai:chat",
"openai_chat_windsurf_headers",
),
)
.await;
return Ok(None);
}
};
request_identity_response_encoding_when_redacted(
&mut provider_request_headers,
request_redacted,
);
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", "openai:chat");
let resolved_report_kind =
if decision_kind == OPENAI_CHAT_STREAM_PLAN_KIND || !upstream_is_stream {
report_kind.to_string()
} else {
"openai_chat_sync_finalize".to_string()
};
Ok(Some(LocalOpenAiChatCandidatePayloadParts {
client_api_format: "openai:chat".to_string(),
auth_header: prepared_candidate.auth_header,
auth_value: prepared_candidate.auth_value,
mapped_model: prepared_candidate.mapped_model,
provider_api_format: "openai:chat".to_string(),
provider_request_body,
provider_request_headers,
upstream_url,
execution_strategy,
conversion_mode,
report_kind: resolved_report_kind,
envelope_name: Some(WINDSURF_ENVELOPE_NAME),
transport: Arc::clone(transport),
request_redacted,
transport_profile: None,
image_request_summary: None,
}))
}
#[allow(clippy::too_many_arguments)]
async fn build_kiro_openai_chat_cross_format_payload_parts(
state: &AppState,

View File

@@ -14,10 +14,7 @@ use crate::ai_serving::planner::candidate_metadata::{
LocalExecutionCandidateMetadataParts,
};
use crate::ai_serving::planner::candidate_resolution::SkippedLocalExecutionCandidate;
use crate::ai_serving::planner::candidate_source::{
preselect_local_execution_candidates_for_api_formats_with_serving,
LocalCandidatePreselectionKeyMode,
};
use crate::ai_serving::planner::candidate_source::LocalCandidatePreselectionKeyMode;
use crate::ai_serving::planner::materialization_policy::{
build_local_candidate_persistence_policy, LocalCandidatePersistencePolicyKind,
};
@@ -26,7 +23,7 @@ use crate::ai_serving::{
ai_local_execution_contract_for_formats, extract_pool_sticky_session_token,
ExecutionRuntimeAuthContext, PlannerAppState,
};
use crate::{AppState, GatewayError};
use crate::AppState;
pub(crate) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalOpenAiChatCandidateAttempt;
pub(crate) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttemptSource as LocalOpenAiChatCandidateAttemptSource;
@@ -357,95 +354,3 @@ pub(crate) async fn build_lazy_local_openai_chat_candidate_attempt_source<'a>(
)
.await
}
pub(crate) async fn build_local_openai_chat_image_candidate_attempt_source<'a>(
state: &'a AppState,
trace_id: &str,
input: &LocalOpenAiChatDecisionInput,
body_json: &serde_json::Value,
) -> Result<(LocalOpenAiChatCandidateAttemptSource<'a>, usize), GatewayError> {
let planner_state = PlannerAppState::new(state);
let sticky_session_token = extract_pool_sticky_session_token(body_json);
let auth_context: &ExecutionRuntimeAuthContext = &input.auth_context;
let persistence_policy = build_local_candidate_persistence_policy(
auth_context,
input.required_capabilities.as_ref(),
LocalCandidatePersistencePolicyKind::OpenAiChatDecision,
);
let preselection = preselect_local_execution_candidates_for_api_formats_with_serving(
planner_state,
"openai:chat",
&input.requested_model,
false,
input.required_capabilities.as_ref(),
&input.auth_snapshot,
input.routing_policy.as_ref(),
input.client_session_affinity.as_ref(),
false,
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
vec!["openai:image".to_string()],
)
.await?;
Ok(build_local_execution_candidate_attempt_source_with_serving(
planner_state,
trace_id,
"openai:chat",
Some(&input.requested_model),
Some(&input.auth_snapshot),
input.client_session_affinity.as_ref(),
input.required_capabilities.as_ref(),
input.routing_policy.as_ref(),
sticky_session_token.as_deref(),
input.request_auth_channel.as_deref(),
persistence_policy,
preselection.candidates,
preselection.skipped_candidates,
LocalCandidateResolutionMode::WithoutTransportPairGate,
|eligible| {
let provider_api_format = eligible.provider_api_format.clone();
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", &provider_api_format);
Some(build_local_execution_candidate_contract_metadata(
LocalExecutionCandidateMetadataParts {
eligible,
provider_api_format: provider_api_format.as_str(),
client_api_format: "openai:chat",
extra_fields: serde_json::Map::new(),
},
execution_strategy,
conversion_mode,
eligible.candidate.endpoint_api_format.trim(),
))
},
|mut skipped_candidate| {
let provider_api_format = skipped_candidate
.transport
.as_ref()
.map(|transport| transport.endpoint.api_format.trim().to_ascii_lowercase())
.unwrap_or_else(|| {
skipped_candidate
.candidate
.endpoint_api_format
.trim()
.to_ascii_lowercase()
});
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", &provider_api_format);
skipped_candidate.extra_data = Some(
build_local_execution_candidate_contract_metadata_for_candidate(
&skipped_candidate.candidate,
skipped_candidate.transport_ref(),
provider_api_format.as_str(),
"openai:chat",
serde_json::Map::new(),
execution_strategy,
conversion_mode,
provider_api_format.as_str(),
),
);
skipped_candidate
},
)
.await)
}

View File

@@ -11,7 +11,6 @@ mod plans;
use self::decision::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
maybe_build_local_openai_chat_decision_payload_for_candidate, LocalOpenAiChatCandidateAttempt,
LocalOpenAiChatCandidateAttemptSource, LocalOpenAiChatDecisionInput,
};

View File

@@ -1,10 +1,8 @@
use async_trait::async_trait;
use tracing::warn;
use super::super::super::openai_request_is_image_generation_intent;
use super::super::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
maybe_build_local_openai_chat_decision_payload_for_candidate, AppState, GatewayControlDecision,
GatewayError, LocalOpenAiChatCandidateAttempt, LocalOpenAiChatCandidateAttemptSource,
LocalOpenAiChatDecisionInput,
@@ -51,53 +49,14 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
};
let effective_body_json = input.effective_body_json(body_json).clone();
let image_generation_intent =
openai_request_is_image_generation_intent(&input.requested_model, body_json);
let (mut candidates, mut candidate_count) = if image_generation_intent {
let (image_candidates, image_candidate_count) =
build_local_openai_chat_image_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
)
.await?;
if image_candidate_count > 0 {
(image_candidates, image_candidate_count)
} else {
build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await
}
} else {
build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await
};
if !image_generation_intent && candidate_count == 0 {
let (image_candidates, image_candidate_count) =
build_local_openai_chat_image_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
)
.await?;
if image_candidate_count > 0 {
candidates = image_candidates;
candidate_count = image_candidate_count;
}
}
let (candidates, candidate_count) = build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await;
if candidate_count == 0 {
set_local_openai_chat_candidate_evaluation_diagnostic(
state,

View File

@@ -1,75 +1,55 @@
pub(crate) fn openai_request_is_image_generation_intent(
requested_model: &str,
body_json: &serde_json::Value,
use serde_json::Value;
pub(super) fn openai_request_is_image_generation_intent(
_requested_model: &str,
body_json: &Value,
) -> bool {
openai_model_is_image_generation(requested_model)
|| body_json
.get("model")
.and_then(serde_json::Value::as_str)
.is_some_and(openai_model_is_image_generation)
|| openai_tool_choice_selects_image_generation(body_json.get("tool_choice"))
request_forces_image_generation_tool(body_json)
}
fn openai_model_is_image_generation(model: &str) -> bool {
model.trim().to_ascii_lowercase().starts_with("gpt-image-")
fn request_forces_image_generation_tool(body_json: &Value) -> bool {
body_json
.get("tool_choice")
.is_some_and(value_is_image_generation_tool)
}
fn openai_tool_choice_selects_image_generation(choice: Option<&serde_json::Value>) -> bool {
let Some(choice) = choice else {
return false;
};
if let Some(value) = choice.as_str() {
return value.trim().eq_ignore_ascii_case("image_generation");
}
let Some(object) = choice.as_object() else {
return false;
};
object
fn value_is_image_generation_tool(value: &Value) -> bool {
value
.get("type")
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
|| object
.get("tool")
.and_then(|value| value.get("type"))
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
|| object
.get("function")
.and_then(|value| value.get("name"))
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
.and_then(Value::as_str)
.is_some_and(|tool_type| tool_type.trim().eq_ignore_ascii_case("image_generation"))
}
#[cfg(test)]
mod tests {
use super::openai_request_is_image_generation_intent;
use serde_json::json;
use super::*;
#[test]
fn detects_openai_image_generation_intent_like_compat_proxies() {
assert!(openai_request_is_image_generation_intent(
"GPT-IMAGE-2",
&json!({})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"model":"gpt-image-2"})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tool_choice":{"function":{"name":"image_generation"}}})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tool_choice":{"type":"image_generation"}})
));
fn tools_declaration_without_tool_choice_does_not_trigger_image_generation() {
let body_json = serde_json::json!({
"model": "gpt-image-2",
"input": "Draw a mountain observatory",
"tools": [{"type": "image_generation"}]
});
assert!(!openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tools":[{"type":"image_generation"}]})
"gpt-image-2",
&body_json
));
assert!(!openai_request_is_image_generation_intent(
"gpt-5",
&json!({"messages":[{"role":"user","content":"hello"}]})
}
#[test]
fn explicit_image_generation_tool_choice_triggers_image_generation() {
let body_json = serde_json::json!({
"model": "gpt-image-2",
"input": "Draw a mountain observatory",
"tools": [{"type": "image_generation"}],
"tool_choice": {"type": "image_generation"}
});
assert!(openai_request_is_image_generation_intent(
"gpt-image-2",
&body_json
));
}
}

View File

@@ -17,7 +17,7 @@ pub(crate) use chat::{
set_local_openai_chat_execution_exhausted_diagnostic,
};
pub(crate) use embedding::maybe_build_sync_local_openai_embedding_decision_payload;
pub(super) use image_intent::openai_request_is_image_generation_intent;
use image_intent::openai_request_is_image_generation_intent;
pub(crate) use responses::{
build_local_openai_responses_stream_attempt_source_for_kind,
build_local_openai_responses_stream_plan_and_reports_for_kind,

View File

@@ -39,10 +39,13 @@ use crate::ai_serving::transport::kiro::{
use crate::ai_serving::transport::{
build_grok_browser_headers, build_grok_upstream_url, build_kiro_cross_format_upstream_url,
build_openai_image_headers, build_openai_image_upstream_url,
build_standard_provider_request_headers,
local_standard_transport_unsupported_reason_with_network,
build_standard_provider_request_headers, build_windsurf_cascade_headers,
build_windsurf_cascade_request_body, build_windsurf_cascade_upstream_url,
is_windsurf_provider_transport, local_standard_transport_unsupported_reason_with_network,
local_windsurf_request_transport_unsupported_reason_with_network,
openai_image_transport_unsupported_reason, resolve_openai_image_auth, GrokHeaderInput,
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
WINDSURF_ENVELOPE_NAME,
};
use crate::ai_serving::{
ai_local_execution_contract_for_formats, request_conversion_direct_auth,
@@ -114,7 +117,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
.trim()
.eq_ignore_ascii_case("grok");
if provider_api_format.eq_ignore_ascii_case("openai:image") {
if !is_grok && provider_api_format.eq_ignore_ascii_case("openai:image") {
return resolve_openai_responses_to_openai_image_payload_parts(
state,
parts,
@@ -128,6 +131,8 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
)
.await;
}
let is_windsurf_cascade =
provider_api_format == "openai:chat" && is_windsurf_provider_transport(transport);
let same_format = api_format_alias_matches(provider_api_format, &client_api_format);
let conversion_kind = request_conversion_kind(spec_metadata.api_format, provider_api_format);
@@ -139,6 +144,8 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
local_kiro_request_transport_unsupported_reason_with_network(transport)
} else if same_format {
local_standard_transport_unsupported_reason_with_network(transport, provider_api_format)
} else if is_windsurf_cascade {
local_windsurf_request_transport_unsupported_reason_with_network(transport)
} else {
match conversion_kind {
Some(_) if is_antigravity && provider_api_format == "gemini:generate_content" => None,
@@ -302,7 +309,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
upstream_is_stream,
force_body_stream_field,
transport.provider.provider_type.as_str(),
if is_kiro_claude_cli {
if is_kiro_claude_cli || is_windsurf_cascade {
None
} else {
transport.endpoint.body_rules.as_ref()
@@ -319,7 +326,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
force_body_stream_field,
transport.provider.provider_type.as_str(),
provider_api_format,
if is_kiro_claude_cli {
if is_kiro_claude_cli || is_windsurf_cascade {
None
} else {
transport.endpoint.body_rules.as_ref()
@@ -447,6 +454,27 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
)
.await;
}
if is_windsurf_cascade {
return build_windsurf_openai_responses_payload_parts(
state,
parts,
trace_id,
body_json,
input,
eligible,
candidate_index,
candidate_id,
spec_metadata.api_format,
transport,
provider_api_format,
mapped_model,
auth_header,
auth_value,
provider_request_body,
upstream_is_stream,
)
.await;
}
let Some(upstream_url) = (if is_grok && is_grok_text_provider_api_format(provider_api_format) {
Some(build_grok_upstream_url(transport, GROK_CHAT_PATH))
@@ -619,6 +647,130 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
})
}
#[allow(clippy::too_many_arguments)]
async fn build_windsurf_openai_responses_payload_parts(
state: &AppState,
parts: &http::request::Parts,
trace_id: &str,
original_body_json: &serde_json::Value,
input: &LocalOpenAiResponsesDecisionInput,
eligible: &EligibleLocalExecutionCandidate,
candidate_index: u32,
candidate_id: &str,
client_api_format: &str,
transport: &Arc<GatewayProviderTransportSnapshot>,
provider_api_format: &str,
mapped_model: String,
auth_header: String,
auth_value: String,
openai_chat_request_body: Value,
upstream_is_stream: bool,
) -> Option<LocalOpenAiResponsesCandidatePayloadParts> {
let candidate = &eligible.candidate;
let effective_headers = input.effective_headers(&parts.headers);
let provider_request_body = match build_windsurf_cascade_request_body(
&openai_chat_request_body,
&mapped_model,
&auth_value,
transport.endpoint.body_rules.as_ref(),
Some(effective_headers),
upstream_is_stream,
) {
Some(body) => body,
None => {
mark_skipped_local_openai_responses_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"provider_request_body_build_failed",
CandidateFailureDiagnostic::envelope_build_failed(
client_api_format,
provider_api_format,
"openai_responses_windsurf_cascade",
),
)
.await;
return None;
}
};
let upstream_url = match build_windsurf_cascade_upstream_url(
transport.endpoint.base_url.as_str(),
parts.uri.query(),
) {
Some(url) => url,
None => {
mark_skipped_local_openai_responses_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"upstream_url_missing",
CandidateFailureDiagnostic::upstream_url_missing(
client_api_format,
provider_api_format,
"openai_responses_windsurf_url",
),
)
.await;
return None;
}
};
let provider_request_headers = match build_windsurf_cascade_headers(
effective_headers,
&provider_request_body,
original_body_json,
transport.endpoint.header_rules.as_ref(),
&auth_header,
&auth_value,
upstream_is_stream,
) {
Some(headers) => headers,
None => {
mark_skipped_local_openai_responses_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"transport_header_rules_apply_failed",
CandidateFailureDiagnostic::header_rules_apply_failed(
client_api_format,
provider_api_format,
"openai_responses_windsurf_headers",
),
)
.await;
return None;
}
};
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
Some(LocalOpenAiResponsesCandidatePayloadParts {
auth_header,
auth_value,
mapped_model,
provider_api_format: provider_api_format.to_string(),
provider_request_body,
provider_request_headers,
upstream_url,
execution_strategy,
conversion_mode,
is_antigravity: false,
envelope_name: Some(WINDSURF_ENVELOPE_NAME),
upstream_is_stream,
transport: Arc::clone(transport),
transport_profile: None,
image_request_summary: None,
})
}
fn api_format_alias_matches(left: &str, right: &str) -> bool {
crate::ai_serving::api_format_alias_matches(left, right)
}
@@ -739,7 +891,11 @@ async fn resolve_openai_responses_to_openai_image_payload_parts(
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
build_openai_image_upstream_url(
transport,
Some("/v1/images/generations"),
parts.uri.query(),
)
};
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {

View File

@@ -20,6 +20,7 @@ pub(crate) use aether_ai_formats::api::{
build_local_openai_responses_request_body,
build_local_openai_responses_request_body_with_model_directives,
build_local_success_background_report, build_local_success_conversion_background_report,
build_openai_image_api_provider_request_body,
build_openai_image_provider_body_from_response_stream_sync_body,
build_openai_image_provider_request_body,
build_openai_image_request_body_from_gemini_image_request,

View File

@@ -50,6 +50,10 @@ pub(crate) mod vertex {
pub(crate) use aether_provider_transport::vertex::*;
}
pub(crate) mod windsurf {
pub(crate) use aether_provider_transport::windsurf::*;
}
pub(crate) use aether_provider_transport::{
append_transport_diagnostics_to_value, apply_local_body_rules,
apply_local_body_rules_with_request_headers, apply_local_header_rules,
@@ -69,12 +73,15 @@ pub(crate) use aether_provider_transport::{
build_standard_plan_fallback_openai_responses_url, build_standard_provider_request_headers,
build_transport_request_url, build_transport_request_url_for_request_body,
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
candidate_common_transport_skip_reason, candidate_transport_pair_skip_reason,
classify_same_format_provider_request_behavior, ensure_upstream_auth_header,
gemini_files_transport_unsupported_reason, header_rules_are_locally_supported,
header_rules_have_enabled_rules, local_gemini_transport_unsupported_reason_with_network,
build_windsurf_cascade_headers, build_windsurf_cascade_request_body,
build_windsurf_cascade_upstream_url, candidate_common_transport_skip_reason,
candidate_transport_pair_skip_reason, classify_same_format_provider_request_behavior,
ensure_upstream_auth_header, gemini_files_transport_unsupported_reason,
header_rules_are_locally_supported, header_rules_have_enabled_rules,
is_windsurf_provider_transport, local_gemini_transport_unsupported_reason_with_network,
local_openai_chat_transport_unsupported_reason,
local_standard_transport_unsupported_reason_with_network,
local_windsurf_request_transport_unsupported_reason_with_network,
openai_image_transport_unsupported_reason, request_conversion_direct_auth,
request_conversion_enabled_for_transport, request_conversion_transport_supported,
request_conversion_transport_unsupported_reason, request_pair_allowed_for_transport,
@@ -97,4 +104,5 @@ pub(crate) use aether_provider_transport::{
StandardPlanFallbackHeadersInput, StandardProviderRequestHeaders,
StandardProviderRequestHeadersInput, TransportRequestBodySemanticsError,
TransportRequestUrlParams, GROK_CHAT_PATH, GROK_INTERNAL_HEADER, GROK_RATE_LIMITS_PATH,
WINDSURF_ENVELOPE_NAME,
};

View File

@@ -8,7 +8,7 @@ use aether_data_contracts::repository::video_tasks::{
use aether_usage_runtime::{build_upsert_usage_record_from_event, settle_usage_if_needed};
use serde_json::{Map, Value};
use tokio::task::JoinHandle;
use tracing::{info, warn};
use tracing::{debug, info, warn};
use crate::log_ids::short_request_id;
use crate::usage::{UsageEvent, UsageEventData, UsageEventType};
@@ -148,8 +148,20 @@ pub(crate) fn spawn_video_task_poller(state: AppState) -> Option<JoinHandle<()>>
let mut interval = tokio::time::interval(config.interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if state
.data
.should_defer_maintenance_for_database_pool_pressure(&mut deferred_since)
{
debug!(
event_name = "video_task_poller_deferred",
log_type = "event",
"gateway video task poller deferred because database pool has no idle reserve"
);
continue;
}
if let Err(err) = poll_video_tasks_once(&state, config.batch_size).await {
warn!(
event_name = "video_task_poller_tick_failed",

View File

@@ -0,0 +1,173 @@
use crate::handlers::shared::{
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
};
use crate::{AppState, GatewayError};
use serde_json::{json, Value};
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
#[derive(Debug, Clone)]
pub(crate) struct BarkPushConfig {
pub(crate) enabled: bool,
pub(crate) device_key: Option<String>,
pub(crate) server_url: String,
pub(crate) template: Option<String>,
}
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
let value = state
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
.await?;
Ok(system_config_bool(value.as_ref(), false))
}
pub(crate) async fn bark_push_configured(state: &AppState) -> Result<bool, GatewayError> {
let config = read_bark_push_config(state).await?;
Ok(config.device_key.is_some() && !config.server_url.trim().is_empty())
}
pub(crate) async fn read_bark_push_config(
state: &AppState,
) -> Result<BarkPushConfig, GatewayError> {
let enabled = bark_push_module_enabled(state).await?;
let device_key = state
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
.await?
.and_then(|value| system_config_string(Some(&value)))
.map(|value| {
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
});
let server_url = state
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
.await?
.and_then(|value| system_config_string(Some(&value)))
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
let template = state
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
.await?
.and_then(|value| system_config_string(Some(&value)));
Ok(BarkPushConfig {
enabled,
device_key,
server_url,
template,
})
}
pub(crate) async fn send_bark_push(
state: &AppState,
config: &BarkPushConfig,
title: &str,
markdown_body: &str,
) -> Result<(), GatewayError> {
let Some(device_key) = config.device_key.as_deref() else {
return Err(GatewayError::Internal("未配置 Bark Device Key".to_string()));
};
let device_key = device_key.trim();
if device_key.is_empty() {
return Err(GatewayError::Internal(
"Bark Device Key 不能为空".to_string(),
));
}
let server_url = normalized_bark_server_url(&config.server_url)?;
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
let response = state
.client
.post(format!("{server_url}/push"))
.json(&json!({
"device_key": device_key,
"title": title,
"body": body,
}))
.send()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let status = response.status();
let text = response
.text()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if !status.is_success() {
return Err(GatewayError::Internal(format!(
"Bark 返回 HTTP {status}: {text}"
)));
}
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
let code_is_ok = payload
.get("code")
.and_then(|value| {
value
.as_i64()
.map(|code| matches!(code, 0 | 200))
.or_else(|| {
value
.as_str()
.map(|code| matches!(code.trim(), "0" | "200"))
})
})
.unwrap_or(true);
if !code_is_ok {
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
}
}
Ok(())
}
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
let server_url = server_url.trim().trim_end_matches('/');
if server_url.is_empty() {
return Err(GatewayError::Internal(
"Bark 服务器地址不能为空".to_string(),
));
}
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
return Err(GatewayError::Internal(
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
));
}
Ok(server_url.to_string())
}
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
match template {
Some(template) if !template.trim().is_empty() => template
.replace("{title}", title)
.replace("{body}", markdown_body),
_ => markdown_body.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::{normalized_bark_server_url, render_bark_body};
#[test]
fn bark_body_uses_template_when_provided() {
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
assert_eq!(rendered, "告警\n\n原始正文");
}
#[test]
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
assert_eq!(
render_bark_body(Some(" "), "告警", "原始正文"),
"原始正文"
);
}
#[test]
fn bark_server_url_trims_trailing_slashes() {
assert_eq!(
normalized_bark_server_url(" https://api.day.app/ ").expect("url should parse"),
"https://api.day.app"
);
}
}

View File

@@ -177,17 +177,6 @@ pub(super) fn classify_admin_endpoints_family_route(
"admin:endpoints_manage",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/endpoints/providers/")
&& normalized_path.ends_with("/key-balance")
{
Some(classified(
"admin_proxy",
"endpoints_manage",
"query_key_balance",
"admin:endpoints_manage",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/endpoints/providers/")
&& normalized_path.ends_with("/keys")

View File

@@ -162,6 +162,16 @@ pub(super) fn classify_admin_system_family_route(
"admin:system",
false,
))
} else if method == http::Method::POST
&& normalized_path == "/api/admin/system/important-notification/test"
{
Some(classified(
"admin_proxy",
"system_manage",
"important_notification_test",
"admin:system",
false,
))
} else if method == http::Method::POST && normalized_path == "/api/admin/system/cleanup" {
Some(classified(
"admin_proxy",

View File

@@ -183,6 +183,10 @@ fn classifies_admin_system_maintenance_write_routes_as_admin_proxy_route() {
("/api/admin/system/users/import", "users_import"),
("/api/admin/system/data/import", "data_import"),
("/api/admin/system/smtp/test", "smtp_test"),
(
"/api/admin/system/important-notification/test",
"important_notification_test",
),
("/api/admin/system/cleanup", "cleanup"),
("/api/admin/system/purge/config", "purge_config"),
("/api/admin/system/purge/users", "purge_users"),

View File

@@ -380,20 +380,6 @@ fn classifies_admin_refresh_provider_quota_as_admin_proxy_route() {
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn classifies_admin_query_provider_key_balance_as_admin_proxy_route() {
let headers = http::HeaderMap::new();
let uri: Uri = "/api/admin/endpoints/providers/provider-newapi/key-balance"
.parse()
.expect("uri should parse");
let decision = classify_control_route(&http::Method::POST, &uri, &headers)
.expect("decision should resolve");
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
assert_eq!(decision.route_family.as_deref(), Some("endpoints_manage"));
assert_eq!(decision.route_kind.as_deref(), Some("query_key_balance"));
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn admin_refresh_provider_quota_buffers_request_body_for_key_selection() {
let headers = headers(&[]);
@@ -413,25 +399,6 @@ fn admin_refresh_provider_quota_buffers_request_body_for_key_selection() {
assert!(local_proxy_route_requires_buffered_body(&context));
}
#[test]
fn admin_query_provider_key_balance_buffers_request_body_for_key_secret() {
let headers = headers(&[]);
let uri: Uri = "/api/admin/endpoints/providers/provider-newapi/key-balance"
.parse()
.expect("uri should parse");
let decision = classify_control_route(&http::Method::POST, &uri, &headers)
.expect("decision should resolve");
let context = GatewayPublicRequestContext::from_request_parts(
"trace-key-balance",
&http::Method::POST,
&uri,
&headers,
Some(decision),
);
assert!(local_proxy_route_requires_buffered_body(&context));
}
#[test]
fn classifies_admin_default_body_rules_as_admin_proxy_route() {
let headers = headers(&[]);

View File

@@ -2000,7 +2000,7 @@ fn resolve_effective_rate_limit_policy(
groups: &[aether_data::repository::users::StoredUserGroup],
) -> Option<i32> {
let group_policy = groups.iter().fold(None, |effective, group| {
intersect_rate_limit_policies(
union_rate_limit_policies(
effective,
rate_limit_restriction_from_mode(&group.rate_limit_mode, group.rate_limit),
)
@@ -2073,6 +2073,22 @@ fn intersect_rate_limit_policies(
}
}
fn union_rate_limit_policies(
left: Option<RateLimitRestriction>,
right: Option<RateLimitRestriction>,
) -> Option<RateLimitRestriction> {
match (left, right) {
(None, None) => None,
(Some(value), None) | (None, Some(value)) => Some(value),
(Some(RateLimitRestriction::Unlimited), _) | (_, Some(RateLimitRestriction::Unlimited)) => {
Some(RateLimitRestriction::Unlimited)
}
(Some(RateLimitRestriction::Limited(left)), Some(RateLimitRestriction::Limited(right))) => {
Some(RateLimitRestriction::Limited(left.max(right)))
}
}
}
fn rate_limit_policy_value(policy: Option<RateLimitRestriction>) -> Option<i32> {
match policy {
None => None,
@@ -2358,35 +2374,44 @@ mod tests {
}
#[test]
fn rate_limit_policy_uses_most_restrictive_custom_limit() {
let groups = vec![sample_group(
"restricted",
10,
None,
"unrestricted",
Some(60),
"custom",
)];
fn rate_limit_policy_uses_highest_group_limit_before_user_restriction() {
let groups = vec![
sample_group("default", 10, None, "unrestricted", Some(30), "custom"),
sample_group("tier-1", 20, None, "unrestricted", Some(100), "custom"),
];
assert_eq!(
resolve_effective_rate_limit_policy(Some(120), "custom", &groups),
Some(60)
Some(100)
);
}
#[test]
fn rate_limit_unlimited_does_not_bypass_limited_group() {
fn rate_limit_unlimited_group_overrides_limited_groups() {
let groups = vec![
sample_group("default", 10, None, "unrestricted", Some(30), "custom"),
sample_group("tier-2", 20, None, "unrestricted", Some(0), "custom"),
];
assert_eq!(
resolve_effective_rate_limit_policy(None, "system", &groups),
Some(0)
);
}
#[test]
fn rate_limit_user_policy_still_restricts_group_grants() {
let groups = vec![sample_group(
"restricted",
"tier-1",
10,
None,
"unrestricted",
Some(60),
Some(100),
"custom",
)];
assert_eq!(
resolve_effective_rate_limit_policy(Some(0), "custom", &groups),
resolve_effective_rate_limit_policy(Some(60), "custom", &groups),
Some(60)
);
}

View File

@@ -42,8 +42,12 @@ use aether_data_contracts::repository::usage::{
};
use aether_runtime_state::RuntimeQueueStore;
use aether_video_tasks_core::read_data_backed_video_task_response;
use std::time::{Duration, Instant};
impl GatewayDataState {
const MAINTENANCE_POOL_IDLE_RESERVE: usize = 1;
const MAINTENANCE_POOL_PRESSURE_MAX_DEFER: Duration = Duration::from_secs(30);
pub(crate) async fn run_database_maintenance(
&self,
table_names: &[&str],
@@ -112,6 +116,47 @@ impl GatewayDataState {
.and_then(|backends| backends.database_pool_summary())
}
pub(crate) fn database_pool_under_maintenance_pressure(&self) -> bool {
self.database_pool_summary()
.as_ref()
.is_some_and(Self::database_pool_summary_under_maintenance_pressure)
}
pub(crate) fn database_pool_summary_under_maintenance_pressure(
summary: &aether_data::DatabasePoolSummary,
) -> bool {
summary.checked_out > 0 && summary.idle <= Self::MAINTENANCE_POOL_IDLE_RESERVE
}
pub(crate) fn should_defer_maintenance_for_database_pool_pressure(
&self,
deferred_since: &mut Option<Instant>,
) -> bool {
Self::should_defer_maintenance_for_pool_pressure_state(
self.database_pool_under_maintenance_pressure(),
deferred_since,
)
}
pub(crate) fn should_defer_maintenance_for_pool_pressure_state(
pool_under_pressure: bool,
deferred_since: &mut Option<Instant>,
) -> bool {
if !pool_under_pressure {
*deferred_since = None;
return false;
}
let now = Instant::now();
let since = deferred_since.get_or_insert(now);
if now.duration_since(*since) >= Self::MAINTENANCE_POOL_PRESSURE_MAX_DEFER {
*deferred_since = None;
return false;
}
true
}
pub(crate) async fn aggregate_wallet_daily_usage(
&self,
input: &WalletDailyUsageAggregationInput,

View File

@@ -1,4 +1,5 @@
use std::sync::Arc;
use std::time::{Duration, Instant};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
@@ -51,6 +52,65 @@ fn disabled_gateway_data_state_has_no_backends() {
assert!(!state.has_video_task_reader());
}
#[test]
fn maintenance_pool_pressure_keeps_idle_reserve_for_foreground_work() {
let pressured = aether_data::DatabasePoolSummary {
driver: DatabaseDriver::Postgres,
checked_out: 6,
pool_size: 6,
idle: 0,
max_connections: 20,
usage_rate: 30.0,
};
assert!(GatewayDataState::database_pool_summary_under_maintenance_pressure(&pressured));
let one_idle_left = aether_data::DatabasePoolSummary {
driver: DatabaseDriver::Postgres,
checked_out: 5,
pool_size: 6,
idle: 1,
max_connections: 20,
usage_rate: 25.0,
};
assert!(GatewayDataState::database_pool_summary_under_maintenance_pressure(&one_idle_left));
let idle = aether_data::DatabasePoolSummary {
driver: DatabaseDriver::Postgres,
checked_out: 0,
pool_size: 4,
idle: 4,
max_connections: 20,
usage_rate: 0.0,
};
assert!(!GatewayDataState::database_pool_summary_under_maintenance_pressure(&idle));
}
#[test]
fn maintenance_pool_pressure_deferral_has_timeout() {
let mut deferred_since = None;
assert!(
GatewayDataState::should_defer_maintenance_for_pool_pressure_state(
true,
&mut deferred_since
)
);
assert!(deferred_since.is_some());
assert!(
!GatewayDataState::should_defer_maintenance_for_pool_pressure_state(
false,
&mut deferred_since
)
);
assert!(deferred_since.is_none());
let mut stale_defer = Some(Instant::now() - Duration::from_secs(31));
assert!(
!GatewayDataState::should_defer_maintenance_for_pool_pressure_state(true, &mut stale_defer)
);
assert!(stale_defer.is_none());
}
#[tokio::test]
async fn postgres_gateway_data_state_builds_video_task_reader() {
let state = GatewayDataState::from_config(GatewayDataConfig::from_postgres_url(

View File

@@ -0,0 +1,381 @@
use base64::Engine;
use crate::handlers::shared::{
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
};
use crate::{AppState, GatewayError};
const SMTP_TIMEOUT_SECS: u64 = 30;
#[derive(Debug, Clone)]
pub(crate) struct SmtpDeliveryConfig {
pub(crate) host: String,
pub(crate) port: u16,
pub(crate) user: Option<String>,
pub(crate) password: Option<String>,
pub(crate) use_tls: bool,
pub(crate) use_ssl: bool,
pub(crate) from_email: String,
pub(crate) from_name: String,
}
#[derive(Debug, Clone)]
pub(crate) struct ComposedEmail {
pub(crate) to_email: String,
pub(crate) subject: String,
pub(crate) html_body: String,
pub(crate) text_body: String,
}
pub(crate) async fn read_smtp_delivery_config(
state: &AppState,
) -> Result<Option<SmtpDeliveryConfig>, GatewayError> {
let smtp_host = state.read_system_config_json_value("smtp_host").await?;
let smtp_from_email = state
.read_system_config_json_value("smtp_from_email")
.await?;
let Some(host) = system_config_string(smtp_host.as_ref()) else {
return Ok(None);
};
let Some(from_email) = system_config_string(smtp_from_email.as_ref()) else {
return Ok(None);
};
let smtp_port = state.read_system_config_json_value("smtp_port").await?;
let smtp_user = state.read_system_config_json_value("smtp_user").await?;
let smtp_password = state.read_system_config_json_value("smtp_password").await?;
let smtp_use_tls = state.read_system_config_json_value("smtp_use_tls").await?;
let smtp_use_ssl = state.read_system_config_json_value("smtp_use_ssl").await?;
let smtp_from_name = state
.read_system_config_json_value("smtp_from_name")
.await?;
let password = system_config_string(smtp_password.as_ref()).map(|value| {
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
});
Ok(Some(SmtpDeliveryConfig {
host,
port: system_config_u16(smtp_port.as_ref(), 587),
user: system_config_string(smtp_user.as_ref()),
password,
use_tls: system_config_bool(smtp_use_tls.as_ref(), true),
use_ssl: system_config_bool(smtp_use_ssl.as_ref(), false),
from_email,
from_name: system_config_string(smtp_from_name.as_ref())
.unwrap_or_else(|| "Aether".to_string()),
}))
}
pub(crate) async fn send_smtp_email(
config: SmtpDeliveryConfig,
email: ComposedEmail,
) -> Result<(), GatewayError> {
tokio::task::spawn_blocking(move || send_smtp_email_blocking(config, email))
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
}
pub(crate) async fn probe_smtp_connection(config: SmtpDeliveryConfig) -> Result<(), GatewayError> {
tokio::task::spawn_blocking(move || probe_smtp_connection_blocking(config))
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
}
pub(crate) fn system_config_u16(value: Option<&serde_json::Value>, default: u16) -> u16 {
match value {
Some(serde_json::Value::Number(value)) => value
.as_u64()
.and_then(|value| u16::try_from(value).ok())
.unwrap_or(default),
Some(serde_json::Value::String(value)) => value.trim().parse::<u16>().unwrap_or(default),
_ => default,
}
}
fn encode_mime_header(value: &str) -> String {
if value.is_ascii() {
return value.to_string();
}
format!(
"=?UTF-8?B?{}?=",
base64::engine::general_purpose::STANDARD.encode(value.as_bytes())
)
}
fn wrap_base64(value: &str) -> String {
let mut wrapped = String::new();
for chunk in value.as_bytes().chunks(76) {
wrapped.push_str(std::str::from_utf8(chunk).unwrap_or_default());
wrapped.push_str("\r\n");
}
wrapped
}
fn build_tls_config() -> std::sync::Arc<rustls::ClientConfig> {
let _ = rustls::crypto::ring::default_provider().install_default();
let root_store =
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
let config = rustls::ClientConfig::builder()
.with_root_certificates(root_store)
.with_no_client_auth();
std::sync::Arc::new(config)
}
fn resolve_server_name(host: &str) -> Result<rustls::pki_types::ServerName<'static>, GatewayError> {
let host = host.trim().trim_start_matches('[').trim_end_matches(']');
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
return Ok(rustls::pki_types::ServerName::from(ip));
}
rustls::pki_types::ServerName::try_from(host.to_string())
.map_err(|err| GatewayError::Internal(err.to_string()))
}
fn connect_tcp_stream(config: &SmtpDeliveryConfig) -> Result<std::net::TcpStream, GatewayError> {
let stream = std::net::TcpStream::connect((config.host.as_str(), config.port))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
stream
.set_read_timeout(Some(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS)))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
stream
.set_write_timeout(Some(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS)))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(stream)
}
fn wrap_tls_stream(
stream: std::net::TcpStream,
host: &str,
) -> Result<rustls::StreamOwned<rustls::ClientConnection, std::net::TcpStream>, GatewayError> {
let server_name = resolve_server_name(host)?;
let connection = rustls::ClientConnection::new(build_tls_config(), server_name)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(rustls::StreamOwned::new(connection, stream))
}
fn smtp_read_response<T: std::io::BufRead>(reader: &mut T) -> Result<(u16, String), GatewayError> {
let mut message = String::new();
let code = loop {
let parsed_code;
let continuation;
let trimmed;
{
let mut line = String::new();
let bytes = reader
.read_line(&mut line)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if bytes == 0 {
return Err(GatewayError::Internal(
"smtp connection closed unexpectedly".to_string(),
));
}
trimmed = line.trim_end_matches(['\r', '\n']).to_string();
if trimmed.len() < 3 {
return Err(GatewayError::Internal("invalid smtp response".to_string()));
}
parsed_code = trimmed[..3]
.parse::<u16>()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
continuation = trimmed.as_bytes().get(3).copied() == Some(b'-');
}
if !message.is_empty() {
message.push('\n');
}
message.push_str(&trimmed);
if !continuation {
break parsed_code;
}
};
Ok((code, message))
}
fn smtp_expect<T: std::io::BufRead>(
reader: &mut T,
allowed_codes: &[u16],
) -> Result<String, GatewayError> {
let (code, message) = smtp_read_response(reader)?;
if allowed_codes.contains(&code) {
return Ok(message);
}
Err(GatewayError::Internal(format!(
"unexpected smtp response {code}: {message}"
)))
}
fn smtp_write_line<T: std::io::Write>(writer: &mut T, line: &str) -> Result<(), GatewayError> {
writer
.write_all(line.as_bytes())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
writer
.write_all(b"\r\n")
.map_err(|err| GatewayError::Internal(err.to_string()))?;
writer
.flush()
.map_err(|err| GatewayError::Internal(err.to_string()))
}
fn smtp_send_command<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
command: &str,
allowed_codes: &[u16],
) -> Result<String, GatewayError> {
smtp_write_line(reader.get_mut(), command)?;
smtp_expect(reader, allowed_codes)
}
fn build_email_message(config: &SmtpDeliveryConfig, email: &ComposedEmail) -> String {
let boundary = format!("aether-{}", uuid::Uuid::new_v4().simple());
let text_body =
wrap_base64(&base64::engine::general_purpose::STANDARD.encode(email.text_body.as_bytes()));
let html_body =
wrap_base64(&base64::engine::general_purpose::STANDARD.encode(email.html_body.as_bytes()));
let from_header = if config.from_name.trim().is_empty() {
format!("<{}>", config.from_email)
} else {
format!(
"{} <{}>",
encode_mime_header(config.from_name.trim()),
config.from_email
)
};
format!(
"From: {from_header}\r\nTo: <{to_email}>\r\nSubject: {subject}\r\nMIME-Version: 1.0\r\nContent-Type: multipart/alternative; boundary=\"{boundary}\"\r\n\r\n--{boundary}\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{text_body}--{boundary}\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{html_body}--{boundary}--\r\n",
to_email = email.to_email,
subject = encode_mime_header(&email.subject),
)
}
fn smtp_authenticate<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &SmtpDeliveryConfig,
) -> Result<(), GatewayError> {
let Some(username) = config
.user
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(());
};
let password = config.password.as_deref().unwrap_or("");
smtp_send_command(reader, "AUTH LOGIN", &[334])?;
smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(username.as_bytes()),
&[334],
)?;
smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(password.as_bytes()),
&[235],
)?;
Ok(())
}
fn smtp_deliver_message<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &SmtpDeliveryConfig,
email: &ComposedEmail,
) -> Result<(), GatewayError> {
smtp_send_command(
reader,
&format!("MAIL FROM:<{}>", config.from_email),
&[250],
)?;
smtp_send_command(
reader,
&format!("RCPT TO:<{}>", email.to_email),
&[250, 251],
)?;
smtp_send_command(reader, "DATA", &[354])?;
let message = build_email_message(config, email);
reader
.get_mut()
.write_all(message.as_bytes())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
reader
.get_mut()
.write_all(b"\r\n.\r\n")
.map_err(|err| GatewayError::Internal(err.to_string()))?;
reader
.get_mut()
.flush()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let _ = smtp_expect(reader, &[250])?;
let _ = smtp_send_command(reader, "QUIT", &[221]);
Ok(())
}
fn smtp_send_message<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &SmtpDeliveryConfig,
email: &ComposedEmail,
) -> Result<(), GatewayError> {
smtp_send_command(reader, "EHLO aether.local", &[250])?;
smtp_authenticate(reader, config)?;
smtp_deliver_message(reader, config, email)
}
fn smtp_probe_connection<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &SmtpDeliveryConfig,
) -> Result<(), GatewayError> {
smtp_send_command(reader, "EHLO aether.local", &[250])?;
smtp_authenticate(reader, config)?;
let _ = smtp_send_command(reader, "QUIT", &[221]);
Ok(())
}
fn send_smtp_email_blocking(
config: SmtpDeliveryConfig,
email: ComposedEmail,
) -> Result<(), GatewayError> {
if config.use_ssl {
let stream = connect_tcp_stream(&config)?;
let tls_stream = wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
let _ = smtp_expect(&mut reader, &[220])?;
return smtp_send_message(&mut reader, &config, &email);
}
let stream = connect_tcp_stream(&config)?;
let mut reader = std::io::BufReader::new(stream);
let _ = smtp_expect(&mut reader, &[220])?;
let _ = smtp_send_command(&mut reader, "EHLO aether.local", &[250])?;
if config.use_tls {
let _ = smtp_send_command(&mut reader, "STARTTLS", &[220])?;
let stream = reader.into_inner();
let tls_stream = wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
return smtp_send_message(&mut reader, &config, &email);
}
smtp_authenticate(&mut reader, &config)?;
smtp_deliver_message(&mut reader, &config, &email)
}
fn probe_smtp_connection_blocking(config: SmtpDeliveryConfig) -> Result<(), GatewayError> {
if config.use_ssl {
let stream = connect_tcp_stream(&config)?;
let tls_stream = wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
let _ = smtp_expect(&mut reader, &[220])?;
return smtp_probe_connection(&mut reader, &config);
}
let stream = connect_tcp_stream(&config)?;
let mut reader = std::io::BufReader::new(stream);
let _ = smtp_expect(&mut reader, &[220])?;
let _ = smtp_send_command(&mut reader, "EHLO aether.local", &[250])?;
if config.use_tls {
let _ = smtp_send_command(&mut reader, "STARTTLS", &[220])?;
let stream = reader.into_inner();
let tls_stream = wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
return smtp_probe_connection(&mut reader, &config);
}
smtp_authenticate(&mut reader, &config)?;
let _ = smtp_send_command(&mut reader, "QUIT", &[221]);
Ok(())
}

View File

@@ -20,6 +20,7 @@ mod stream_pump;
pub(crate) mod submission;
pub(crate) mod sync;
pub(crate) mod transport;
mod windsurf;
pub(crate) use self::chatgpt_web_image::maybe_execute_chatgpt_web_image_sync;
pub(crate) use self::constants::{

File diff suppressed because it is too large Load Diff

View File

@@ -126,6 +126,54 @@ pub(super) fn build_stream_failure_from_execution_error(
}
}
pub(super) fn build_stream_failure_from_provider_error_body(
status_code: u16,
body_json: &Value,
) -> StreamFailureReport {
let body_object = body_json.as_object();
let error_object = body_object
.and_then(|object| object.get("error"))
.and_then(Value::as_object);
let error_type =
first_non_empty_error_text(error_object, body_object, &["type", "code", "status"])
.unwrap_or_else(|| "upstream_error".to_string());
let error_message = first_non_empty_error_text(
error_object,
body_object,
&["message", "detail", "reason", "status", "type", "code"],
)
.unwrap_or_else(|| format!("upstream stream returned error status {status_code}"));
StreamFailureReport {
status_code,
error_type,
error_message,
extra_error_fields: Map::new(),
}
}
fn first_non_empty_error_text(
error_object: Option<&Map<String, Value>>,
body_object: Option<&Map<String, Value>>,
keys: &[&str],
) -> Option<String> {
for object in [error_object, body_object].into_iter().flatten() {
for key in keys {
let Some(value) = object.get(*key) else {
continue;
};
match value {
Value::String(text) if !text.trim().is_empty() => {
return Some(text.trim().to_string());
}
Value::Number(number) => return Some(number.to_string()),
_ => {}
}
}
}
None
}
fn build_stream_failure_sync_payload(
trace_id: &str,
report_kind: String,
@@ -296,6 +344,49 @@ async fn record_stream_sync_failure(
.await;
}
#[allow(clippy::too_many_arguments)] // internal helper for prefetch error handling
pub(super) async fn handle_prefetch_provider_private_stream_error(
state: &AppState,
trace_id: &str,
decision: &GatewayControlDecision,
plan: &ExecutionPlan,
report_context: Option<Value>,
request_id: &str,
candidate_id: Option<&str>,
report_kind: &str,
mut headers: std::collections::BTreeMap<String, String>,
telemetry: Option<ExecutionTelemetry>,
buffered_body: &[u8],
status_code: u16,
body_json: Value,
) -> Result<Option<Response<Body>>, GatewayError> {
headers.remove("content-encoding");
headers.remove("content-length");
headers.insert("content-type".to_string(), "application/json".to_string());
let payload = GatewaySyncReportRequest {
trace_id: trace_id.to_string(),
report_kind: report_kind.to_string(),
report_context,
status_code,
headers,
body_json: Some(body_json),
client_body_json: None,
body_base64: (!buffered_body.is_empty())
.then(|| base64::engine::general_purpose::STANDARD.encode(buffered_body)),
telemetry,
};
record_stream_sync_failure(state, plan, payload.report_context.as_ref(), &payload, None).await;
let response =
submit_local_core_error_or_sync_finalize(state, trace_id, decision, payload).await?;
Ok(Some(attach_control_metadata_headers(
response,
Some(request_id),
candidate_id,
)?))
}
#[allow(clippy::too_many_arguments)] // internal helper for prefetch error handling
pub(super) async fn handle_prefetch_stream_failure(
state: &AppState,

View File

@@ -1,9 +1,7 @@
#[cfg(test)]
use crate::ai_serving::api::core_success_background_report_kind;
use crate::ai_serving::api::{
build_core_error_body_for_client_format, core_error_background_report_kind,
core_error_default_client_api_format, is_core_error_finalize_kind,
maybe_compile_sync_finalize_response,
core_error_default_client_api_format, core_success_background_report_kind,
is_core_error_finalize_kind, maybe_compile_sync_finalize_response,
normalize_provider_private_response_value as unwrap_local_finalize_response_value,
LocalCoreSyncErrorKind,
};
@@ -36,6 +34,12 @@ pub(super) fn maybe_build_local_core_error_response(
return Ok(None);
};
let status_source_json = resolve_local_sync_source_body_json(payload)?;
if payload.status_code < 400
&& !has_nested_error(&response_body_json)
&& !status_source_json.as_ref().is_some_and(has_nested_error)
{
return Ok(None);
}
let mut response_headers = payload.headers.clone();
response_headers.remove("content-encoding");
@@ -81,8 +85,9 @@ fn build_local_sync_response_from_json(
payload: &GatewaySyncReportRequest,
body_json: serde_json::Value,
) -> Result<Response<Body>, GatewayError> {
let status_code = if is_core_error_finalize_kind(payload.report_kind.as_str())
|| has_nested_error(&body_json)
let body_has_error = has_nested_error(&body_json);
let status_code = if body_has_error
|| (payload.status_code >= 400 && is_core_error_finalize_kind(payload.report_kind.as_str()))
{
resolve_local_sync_error_status_code(payload.status_code, &body_json)
} else {
@@ -367,7 +372,10 @@ pub(crate) fn resolve_core_success_background_report_kind(report_kind: &str) ->
core_success_background_report_kind(report_kind).map(ToOwned::to_owned)
}
fn resolve_local_sync_error_status_code(status_code: u16, body_json: &serde_json::Value) -> u16 {
pub(crate) fn resolve_local_sync_error_status_code(
status_code: u16,
body_json: &serde_json::Value,
) -> u16 {
if (400..600).contains(&status_code) {
return status_code;
}
@@ -621,11 +629,30 @@ pub(crate) async fn submit_local_core_error_or_sync_finalize(
build_local_core_sync_finalize_fallback_response(trace_id, decision, &payload)?
};
if let Some(error_report_kind) =
let response_status = response.status();
if response_status.is_success() {
if let Some(success_report_kind) =
core_success_background_report_kind(payload.report_kind.as_str())
{
let mut report_payload = payload.clone();
report_payload.report_kind = success_report_kind.to_string();
report_payload.status_code = response_status.as_u16();
spawn_sync_report(state.clone(), report_payload);
} else {
warn!(
event_name = "local_core_finalize_missing_success_report_mapping",
log_type = "event",
trace_id = %trace_id,
report_kind = %payload.report_kind,
"gateway built local core finalize success response without background success report mapping"
);
}
} else if let Some(error_report_kind) =
resolve_core_error_background_report_kind(payload.report_kind.as_str())
{
let mut report_payload = payload.clone();
report_payload.report_kind = error_report_kind;
report_payload.status_code = response_status.as_u16();
spawn_sync_report(state.clone(), report_payload);
} else {
warn!(
@@ -819,4 +846,39 @@ mod tests {
"unexpected message: {message}"
);
}
#[tokio::test]
async fn submit_local_core_finalize_keeps_http_200_for_success_image_body() {
let payload = core_finalize_payload(
"openai_image_sync_finalize",
"openai:image",
"openai:image",
200,
json!({
"created": 1779273523,
"data": [{
"b64_json": "aGVsbG8="
}]
}),
);
let state = AppState::new().expect("state should build");
let response = submit_local_core_error_or_sync_finalize(
&state,
"trace-image-success-200",
&test_decision(),
payload,
)
.await
.expect("response should build");
assert_eq!(response.status(), http::StatusCode::OK);
let body: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("body should decode");
assert_eq!(body["data"][0]["b64_json"], "aGVsbG8=");
}
}

View File

@@ -3,6 +3,7 @@ use std::io::Error as IoError;
use std::sync::Arc;
use std::time::{Duration, Instant};
use aether_ai_serving::UPSTREAM_IS_STREAM_KEY;
use aether_contracts::{
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionPlan, ExecutionResult,
ExecutionTelemetry,
@@ -20,7 +21,6 @@ use async_stream::stream;
use axum::body::{to_bytes, Body, Bytes};
use axum::http::header::{CACHE_CONTROL, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE};
use axum::http::{HeaderName, HeaderValue, Response, StatusCode};
use base64::Engine as _;
use futures_util::StreamExt;
use serde_json::{json, Value};
use tokio::sync::mpsc;
@@ -29,8 +29,9 @@ use tokio::time::MissedTickBehavior;
use tracing::{debug, warn};
use crate::ai_serving::api::{
build_core_error_body_for_client_format, implicit_sync_finalize_report_kind,
maybe_build_sync_finalize_outcome, LocalCoreSyncErrorKind, LocalCoreSyncFinalizeOutcome,
build_core_error_body_for_client_format, extract_provider_private_stream_error_body,
implicit_sync_finalize_report_kind, maybe_build_sync_finalize_outcome, LocalCoreSyncErrorKind,
LocalCoreSyncFinalizeOutcome,
};
use crate::api::response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
@@ -43,12 +44,16 @@ use crate::execution_runtime::grok::maybe_execute_grok_sync;
use crate::execution_runtime::oauth_retry::refresh_oauth_plan_auth_for_retry;
#[cfg(test)]
use crate::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_runtime;
use crate::execution_runtime::submission::submit_local_core_error_or_sync_finalize;
use crate::execution_runtime::transport::{
build_request_body, collect_response_headers, decode_response_body_bytes,
format_upstream_request_error, format_wreq_upstream_request_error, response_body_is_json,
send_request, DirectHttpResponse, DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
use crate::execution_runtime::submission::{
resolve_local_sync_error_status_code, submit_local_core_error_or_sync_finalize,
};
use crate::execution_runtime::transport::{
build_execution_response_body, build_request_body, collect_response_headers,
decode_response_body_bytes, format_upstream_request_error, format_wreq_upstream_request_error,
response_body_is_json, send_request, DirectHttpResponse, DirectSyncExecutionRuntime,
ExecutionRuntimeTransportError,
};
use crate::execution_runtime::windsurf::maybe_execute_windsurf_sync;
use crate::execution_runtime::{
analyze_local_candidate_failover_sync, apply_endpoint_response_header_rules,
attach_provider_response_headers_to_report_context, local_failover_response_text,
@@ -398,6 +403,43 @@ fn build_invalid_provider_success_body(
)
}
fn provider_private_error_details(body_json: &Value) -> (Option<String>, Option<String>) {
let body_object = body_json.as_object();
let error_object = body_object
.and_then(|object| object.get("error"))
.and_then(Value::as_object);
let error_type =
first_non_empty_error_text(error_object, body_object, &["type", "code", "status"]);
let error_message = first_non_empty_error_text(
error_object,
body_object,
&["message", "detail", "reason", "status", "type", "code"],
);
(error_type, error_message)
}
fn first_non_empty_error_text(
error_object: Option<&serde_json::Map<String, Value>>,
body_object: Option<&serde_json::Map<String, Value>>,
keys: &[&str],
) -> Option<String> {
for object in [error_object, body_object].into_iter().flatten() {
for key in keys {
let Some(value) = object.get(*key) else {
continue;
};
match value {
Value::String(text) if !text.trim().is_empty() => {
return Some(text.trim().to_string());
}
Value::Number(number) => return Some(number.to_string()),
_ => {}
}
}
}
None
}
#[derive(Debug, Clone)]
struct OpenAiImageSyncProgressSnapshot {
phase: &'static str,
@@ -788,6 +830,12 @@ async fn execute_direct_sync_runtime_candidate(
candidate_index: &str,
progress_snapshot: Option<Arc<Mutex<OpenAiImageSyncProgressSnapshot>>>,
) -> Result<ExecutionResult, SyncExecutionFailure> {
if let Some(result) = maybe_execute_windsurf_sync(state, plan, report_context)
.await
.map_err(SyncExecutionFailure::from_transport)?
{
return Ok(result);
}
if !should_track_openai_image_sync_upstream_sse(plan_kind, plan, report_context) {
return DirectSyncExecutionRuntime::new()
.execute_sync(plan)
@@ -945,27 +993,9 @@ async fn execute_openai_image_sync_upstream_sse_candidate(
let upstream_bytes = body_bytes.len() as u64;
progress.finish(status_code, elapsed_ms).await;
let body = if body_bytes.is_empty() {
None
} else if plan.stream {
Some(aether_contracts::ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
} else if response_body_is_json(&headers, &decoded_body_bytes) {
let body_json: Value = serde_json::from_slice(&decoded_body_bytes)
.map_err(ExecutionRuntimeTransportError::InvalidJson)
let body =
build_execution_response_body(&headers, &body_bytes, &decoded_body_bytes, plan.stream)
.map_err(SyncExecutionFailure::from_transport)?;
Some(aether_contracts::ResponseBody {
json_body: Some(body_json),
body_bytes_b64: None,
})
} else {
Some(aether_contracts::ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
};
Ok(ExecutionResult {
request_id: plan.request_id.clone(),
@@ -1005,7 +1035,7 @@ fn resolve_openai_image_sync_total_timeout_ms(plan: &ExecutionPlan) -> u64 {
fn report_context_upstream_is_stream(report_context: Option<&Value>) -> bool {
report_context
.and_then(|value| value.get("upstream_is_stream"))
.and_then(|value| value.get(UPSTREAM_IS_STREAM_KEY))
.and_then(Value::as_bool)
.unwrap_or(false)
}
@@ -1696,8 +1726,21 @@ async fn execute_execution_runtime_sync_impl(
headers.insert("content-type".to_string(), "application/json".to_string());
}
}
let (result_error_type, result_error_message) =
let (mut result_error_type, mut result_error_message) =
execution_error_details(result.error.as_ref(), body_json.as_ref());
if result.status_code < 400 && body_json.is_none() {
if let Some(error_body_json) =
extract_provider_private_stream_error_body(report_context.as_ref(), &body_bytes)
{
result.status_code =
resolve_local_sync_error_status_code(result.status_code, &error_body_json);
let (private_error_type, private_error_message) =
provider_private_error_details(&error_body_json);
result_error_type = private_error_type.or(result_error_type);
result_error_message = private_error_message.or(result_error_message);
body_json = Some(error_body_json);
}
}
let local_failover_response_text = local_failover_response_text(
body_json.as_ref(),
&body_bytes,

View File

@@ -25,8 +25,10 @@ use serde_json::json;
use serde_json::Value;
use thiserror::Error;
use crate::ai_serving::api::extract_provider_private_stream_error_body;
#[cfg(test)]
use crate::execution_runtime::remote_compat::execute_sync_plan_via_remote_execution_runtime;
use crate::execution_runtime::windsurf::maybe_execute_windsurf_sync;
use crate::frontdoor_loop_guard::{
configured_gateway_frontdoor_base_url, gateway_frontdoor_self_loop_guard_error,
};
@@ -232,26 +234,8 @@ impl DirectSyncExecutionRuntime {
let elapsed_ms = started_at.elapsed().as_millis() as u64;
let upstream_bytes = body_bytes.len() as u64;
let body = if body_bytes.is_empty() {
None
} else if plan.stream {
Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
} else if response_body_is_json(&headers, &decoded_body_bytes) {
let body_json: Value = serde_json::from_slice(&decoded_body_bytes)
.map_err(ExecutionRuntimeTransportError::InvalidJson)?;
Some(ResponseBody {
json_body: Some(body_json),
body_bytes_b64: None,
})
} else {
Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
};
let body =
build_execution_response_body(&headers, &body_bytes, &decoded_body_bytes, plan.stream)?;
Ok(ExecutionResult {
request_id: plan.request_id.clone(),
@@ -347,6 +331,11 @@ pub(crate) async fn execute_sync_plan_with_report_context(
}
let _ = trace_id;
match maybe_execute_windsurf_sync(state, plan, None).await {
Ok(Some(result)) => return Ok(result),
Ok(None) => {}
Err(err) => return Err(GatewayError::Internal(err.to_string())),
}
match DirectSyncExecutionRuntime::new().execute_sync(plan).await {
Ok(result) => {
record_manual_proxy_request_outcome(state, plan, result.status_code).await;
@@ -567,26 +556,8 @@ async fn execute_sync_plan_via_local_tunnel(
);
}
let body = if body_bytes.is_empty() {
None
} else if plan.stream {
Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
} else if response_body_is_json(&headers, &decoded_body_bytes) {
let body_json: Value = serde_json::from_slice(&decoded_body_bytes)
.map_err(ExecutionRuntimeTransportError::InvalidJson)?;
Some(ResponseBody {
json_body: Some(body_json),
body_bytes_b64: None,
})
} else {
Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(&body_bytes)),
})
};
let body =
build_execution_response_body(&headers, &body_bytes, &decoded_body_bytes, plan.stream)?;
Ok(ExecutionResult {
request_id: plan.request_id.clone(),
@@ -1470,17 +1441,63 @@ pub(crate) fn decode_response_body_bytes(
}
pub(crate) fn response_body_is_json(headers: &BTreeMap<String, String>, body_bytes: &[u8]) -> bool {
if headers
let content_type = headers
.get("content-type")
.map(|value| value.to_ascii_lowercase())
.is_some_and(|value| value.contains("json"))
.unwrap_or_default();
if content_type.contains("application/connect+json")
|| content_type.contains("application/connect+proto")
{
return false;
}
if content_type.contains("json") {
return true;
}
serde_json::from_slice::<Value>(body_bytes).is_ok()
}
pub(crate) fn build_execution_response_body(
headers: &BTreeMap<String, String>,
body_bytes: &[u8],
decoded_body_bytes: &[u8],
stream: bool,
) -> Result<Option<ResponseBody>, ExecutionRuntimeTransportError> {
if body_bytes.is_empty() {
return Ok(None);
}
if let Some(body_json) = extract_provider_private_stream_error_body(None, decoded_body_bytes)
.or_else(|| extract_provider_private_stream_error_body(None, body_bytes))
{
return Ok(Some(ResponseBody {
json_body: Some(body_json),
body_bytes_b64: None,
}));
}
if stream {
return Ok(Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(body_bytes)),
}));
}
if response_body_is_json(headers, decoded_body_bytes) {
let body_json: Value = serde_json::from_slice(decoded_body_bytes)
.map_err(ExecutionRuntimeTransportError::InvalidJson)?;
return Ok(Some(ResponseBody {
json_body: Some(body_json),
body_bytes_b64: None,
}));
}
Ok(Some(ResponseBody {
json_body: None,
body_bytes_b64: Some(base64::engine::general_purpose::STANDARD.encode(body_bytes)),
}))
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -1505,11 +1522,12 @@ mod tests {
use tokio::sync::watch;
use super::{
build_browser_wreq_client, build_client, build_request_headers, execute_sync_plan,
record_manual_proxy_request_failure, record_manual_proxy_request_outcome,
record_manual_proxy_request_success, record_manual_proxy_stream_error,
resolve_execution_transport_controls, DirectSyncExecutionRuntime,
ExecutionRuntimeTransportError, ExecutionTransportControls,
build_browser_wreq_client, build_client, build_execution_response_body,
build_request_headers, execute_sync_plan, record_manual_proxy_request_failure,
record_manual_proxy_request_outcome, record_manual_proxy_request_success,
record_manual_proxy_stream_error, resolve_execution_transport_controls,
response_body_is_json, DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
ExecutionTransportControls,
};
use crate::constants::{
EXECUTION_RUNTIME_LOOP_GUARD_HEADER, EXECUTION_RUNTIME_LOOP_GUARD_VIA_TOKEN,
@@ -2764,6 +2782,41 @@ mod tests {
));
}
#[test]
fn connect_json_response_is_not_treated_as_plain_json() {
let headers = BTreeMap::from([(
"content-type".to_string(),
"application/connect+json".to_string(),
)]);
let body = [2, 0, 0, 0, 2, b'{', b'}'];
assert!(!response_body_is_json(&headers, &body));
}
#[test]
fn connect_json_error_response_is_decoded_for_stream_sync_body() {
let headers = BTreeMap::from([(
"content-type".to_string(),
"application/connect+json".to_string(),
)]);
let payload = br#"{"error":{"code":"resource_exhausted","message":"quota exhausted"}}"#;
let mut body_bytes = vec![2];
body_bytes.extend_from_slice(&(payload.len() as u32).to_be_bytes());
body_bytes.extend_from_slice(payload);
let body = build_execution_response_body(&headers, &body_bytes, &body_bytes, true)
.expect("body should build")
.expect("body should be present");
assert_eq!(
body.json_body
.as_ref()
.and_then(|value| value.pointer("/error/code")),
Some(&json!("resource_exhausted"))
);
assert!(body.body_bytes_b64.is_none());
}
#[tokio::test]
async fn direct_sync_execution_runtime_compresses_json_body_when_requested() {
let listener = crate::test_support::bind_loopback_listener()

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,6 @@
use aether_ai_serving::{
run_ai_attempt_loop, AiAttemptLoopOutcome, AiAttemptLoopPort, AiExecutionAttempt,
UPSTREAM_IS_STREAM_KEY,
};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_scheduler_core::{
@@ -471,11 +472,24 @@ fn should_skip_unused_persistence_from_metadata(
metadata.candidate_group_id.is_some() && metadata.pool_key_index.is_some()
}
fn resolve_stream_candidate_watchdog_timeout(plan: &aether_contracts::ExecutionPlan) -> Duration {
fn resolve_stream_candidate_watchdog_timeout(
plan: &aether_contracts::ExecutionPlan,
report_context: Option<&serde_json::Value>,
) -> Duration {
let upstream_is_stream = report_context
.and_then(|context| context.get(UPSTREAM_IS_STREAM_KEY))
.and_then(serde_json::Value::as_bool)
.unwrap_or(true);
let timeout_ms = plan
.timeouts
.as_ref()
.and_then(|timeouts| timeouts.first_byte_ms.or(timeouts.total_ms))
.and_then(|timeouts| {
if upstream_is_stream {
timeouts.first_byte_ms.or(timeouts.total_ms)
} else {
timeouts.total_ms.or(timeouts.first_byte_ms)
}
})
.unwrap_or(DEFAULT_STREAM_CANDIDATE_WATCHDOG_TIMEOUT_MS)
.max(1);
Duration::from_millis(timeout_ms)
@@ -493,7 +507,7 @@ where
Fut:
std::future::Future<Output = Result<Option<Response<Body>>, GatewayError>> + Send + 'static,
{
let timeout_duration = resolve_stream_candidate_watchdog_timeout(plan);
let timeout_duration = resolve_stream_candidate_watchdog_timeout(plan, report_context);
let candidate_started_unix_ms = current_unix_ms();
let mut join_handle = tokio::spawn(execute());
match timeout(timeout_duration, &mut join_handle).await {
@@ -655,19 +669,22 @@ mod tests {
#[test]
fn stream_candidate_watchdog_prefers_first_byte_timeout() {
let timeout =
resolve_stream_candidate_watchdog_timeout(&test_plan(Some(ExecutionTimeouts {
let report_context = json!({"upstream_is_stream": true});
let timeout = resolve_stream_candidate_watchdog_timeout(
&test_plan(Some(ExecutionTimeouts {
first_byte_ms: Some(12_345),
total_ms: Some(90_000),
..ExecutionTimeouts::default()
})));
})),
Some(&report_context),
);
assert_eq!(timeout, Duration::from_millis(12_345));
}
#[test]
fn stream_candidate_watchdog_uses_default_when_timeouts_missing() {
let timeout = resolve_stream_candidate_watchdog_timeout(&test_plan(None));
let timeout = resolve_stream_candidate_watchdog_timeout(&test_plan(None), None);
assert_eq!(
timeout,
@@ -675,6 +692,50 @@ mod tests {
);
}
#[test]
fn stream_candidate_watchdog_prefers_total_timeout_when_upstream_non_stream() {
let report_context = json!({"upstream_is_stream": false});
let timeout = resolve_stream_candidate_watchdog_timeout(
&test_plan(Some(ExecutionTimeouts {
first_byte_ms: Some(300_000),
total_ms: Some(599_000),
..ExecutionTimeouts::default()
})),
Some(&report_context),
);
assert_eq!(timeout, Duration::from_millis(599_000));
}
#[test]
fn stream_candidate_watchdog_falls_back_to_first_byte_when_upstream_non_stream_lacks_total() {
let report_context = json!({"upstream_is_stream": false});
let timeout = resolve_stream_candidate_watchdog_timeout(
&test_plan(Some(ExecutionTimeouts {
first_byte_ms: Some(300_000),
..ExecutionTimeouts::default()
})),
Some(&report_context),
);
assert_eq!(timeout, Duration::from_millis(300_000));
}
#[test]
fn stream_candidate_watchdog_defaults_to_streaming_when_flag_missing() {
let report_context = json!({});
let timeout = resolve_stream_candidate_watchdog_timeout(
&test_plan(Some(ExecutionTimeouts {
first_byte_ms: Some(12_345),
total_ms: Some(90_000),
..ExecutionTimeouts::default()
})),
Some(&report_context),
);
assert_eq!(timeout, Duration::from_millis(12_345));
}
#[test]
fn unused_persistence_skips_pool_internal_candidates() {
assert!(should_skip_unused_persistence(Some(&json!({

View File

@@ -37,6 +37,7 @@ pub(crate) use self::provider::oauth::runtime::{
refresh_provider_oauth_account_state_after_update,
};
pub(crate) use self::provider::ops::providers::actions::admin_provider_ops_local_action_response;
pub(crate) use self::provider::ops::providers::store_admin_provider_ops_balance_cache;
pub(crate) use self::provider::pool::config::admin_provider_pool_config;
pub(crate) use self::provider::pool_admin::maybe_build_local_admin_pool_response;
pub(crate) use self::provider::shared::payloads::{

View File

@@ -59,6 +59,11 @@ pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
) -> Result<Response<Body>, GatewayError> {
let mut categories = Vec::with_capacity(ADMIN_MONITORING_REDIS_CACHE_CATEGORIES.len());
let mut total_keys = 0usize;
let diagnostics = state
.runtime_state()
.redis_diagnostics()
.await
.map_err(|err| GatewayError::Internal(format!("redis diagnostics failed: {err}")))?;
for (key, name, pattern, description) in ADMIN_MONITORING_REDIS_CACHE_CATEGORIES {
let count = list_admin_monitoring_namespaced_keys(state, pattern)
@@ -81,6 +86,7 @@ pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
"backend": state.runtime_state().backend_kind().as_str(),
"categories": categories,
"total_keys": total_keys,
"diagnostics": diagnostics,
}
}))
.into_response())

View File

@@ -1194,6 +1194,7 @@ async fn admin_monitoring_redis_keys_returns_local_payload_without_redis() {
assert_eq!(payload["data"]["available"], json!(true));
assert_eq!(payload["data"]["backend"], json!("memory"));
assert_eq!(payload["data"]["total_keys"], json!(0));
assert_eq!(payload["data"]["diagnostics"], serde_json::Value::Null);
}
#[tokio::test]

View File

@@ -5,8 +5,11 @@ use super::local_monitoring_response;
use crate::data::GatewayDataState;
use crate::AppState;
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_data_contracts::repository::{
candidates::RequestCandidateStatus, usage::UsageBodyCaptureState,
};
use axum::body::to_bytes;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use serde_json::json;
use std::sync::Arc;
@@ -530,6 +533,89 @@ async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_respon
assert!(extra.get("provider_response").is_none());
}
#[tokio::test]
async fn admin_monitoring_trace_request_decodes_connect_json_response_body_refs() {
let mut candidate = sample_candidate(
"cand-used",
"request-connect",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(429),
);
candidate.extra_data = Some(json!({"cache_1h": true}));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"request-connect",
"provider-1",
"Windsurf",
0,
0.0,
"failed",
Some(429),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.response_headers = Some(json!({
"content-type": "application/connect+json"
}));
let mut framed = Vec::new();
framed.push(2);
let payload = br#"{"error":{"code":"resource_exhausted","message":"quota exhausted"}}"#;
framed.extend_from_slice(&(payload.len() as u32).to_be_bytes());
framed.extend_from_slice(payload);
usage.response_body = Some(json!(BASE64_STANDARD.encode(framed)));
usage.response_body_ref = Some("usage://request/request-connect/response_body".to_string());
usage.response_body_state = Some(UsageBodyCaptureState::Inline);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/request-connect",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
let upstream_response = &payload["candidates"][0]["extra_data"]["upstream_response"];
assert_eq!(upstream_response["status_code"], json!(429));
assert_eq!(
upstream_response["body"]["error"]["code"],
json!("resource_exhausted")
);
assert_eq!(
upstream_response["body"]["error"]["message"],
json!("quota exhausted")
);
assert_eq!(
upstream_response["body_ref"],
json!("usage://request/request-connect/response_body")
);
assert_eq!(upstream_response["body_state"], json!("inline"));
}
#[tokio::test]
async fn admin_monitoring_trace_request_exposes_structured_ranking_metadata() {
let mut candidate = sample_candidate(

View File

@@ -1,4 +1,3 @@
mod balance;
mod mutations;
mod quota;
mod reads;
@@ -19,10 +18,6 @@ pub(crate) async fn maybe_build_local_admin_endpoints_keys_response(
return Ok(Some(response));
}
if let Some(response) = balance::maybe_handle(state, request_context, request_body).await? {
return Ok(Some(response));
}
if let Some(response) = mutations::maybe_handle(state, request_context, request_body).await? {
return Ok(Some(response));
}

View File

@@ -1,6 +1,6 @@
use crate::handlers::admin::request::AdminAppState;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use std::time::{SystemTime, UNIX_EPOCH};

View File

@@ -25,10 +25,15 @@ use crate::handlers::admin::provider::oauth::runtime::{
use crate::handlers::admin::provider::oauth::state::{
admin_provider_oauth_template, exchange_admin_provider_oauth_refresh_token,
};
use crate::handlers::admin::provider::shared::support::ADMIN_PROVIDER_OAUTH_DATA_UNAVAILABLE_DETAIL;
use crate::handlers::admin::request::{AdminAppState, AdminProviderOAuthTemplate};
use crate::GatewayError;
use aether_admin::provider::oauth::parse_admin_provider_oauth_kiro_batch_import_entries;
use aether_contracts::ProxySnapshot;
use aether_oauth::core::OAuthError;
use aether_oauth::provider::{
ProviderOAuthImportInput, ProviderOAuthService, ProviderOAuthTransportContext,
};
use serde_json::{json, Map, Value};
struct AdminProviderOAuthResolvedBatchImport {
@@ -37,6 +42,16 @@ struct AdminProviderOAuthResolvedBatchImport {
expires_at: Option<u64>,
}
fn sanitize_windsurf_batch_import_error(error: &OAuthError) -> String {
match error {
OAuthError::InvalidRequest(_) => "Windsurf 凭据验证失败: 请求参数无效".to_string(),
OAuthError::HttpStatus { status_code, .. } => {
format!("Windsurf 凭据验证失败: HTTP {status_code}")
}
_ => "Windsurf 凭据验证失败".to_string(),
}
}
pub(super) fn estimate_admin_provider_oauth_batch_import_total(
provider_type: &str,
raw_credentials: &str,
@@ -98,6 +113,61 @@ async fn resolve_admin_provider_oauth_batch_import_tokens(
.map(str::trim)
.filter(|value| !value.is_empty());
if provider_type.eq_ignore_ascii_case("windsurf") {
let token_for_import = refresh_token.or(access_token);
let ctx = ProviderOAuthTransportContext {
provider_id: String::new(),
provider_type: provider_type.to_string(),
endpoint_id: None,
key_id: None,
auth_type: Some("oauth".to_string()),
decrypted_api_key: None,
decrypted_auth_config: None,
provider_config: None,
endpoint_config: None,
key_config: None,
network: aether_oauth::network::OAuthNetworkContext::provider_operation(
request_proxy.clone(),
),
};
let executor = crate::oauth::GatewayOAuthHttpExecutor::new(*state);
let result = ProviderOAuthService::with_builtin_adapters()
.import_credentials(
&executor,
&ctx,
ProviderOAuthImportInput {
provider_type: provider_type.to_string(),
name: entry
.raw_credentials
.as_ref()
.and_then(|raw| raw.get("name"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
refresh_token: token_for_import.map(ToOwned::to_owned),
raw_credentials: entry.raw_credentials.clone(),
network: ctx.network.clone(),
},
)
.await
.map_err(|error| sanitize_windsurf_batch_import_error(&error))?;
let access_token = result.token_set.access_token.trim().to_string();
if access_token.is_empty() {
return Err("Windsurf 凭据验证返回缺少 apiKey/sessionToken".to_string());
}
let auth_config = result
.auth_config
.as_object()
.cloned()
.ok_or_else(|| "Windsurf 凭据验证返回缺少 auth_config".to_string())?;
return Ok(AdminProviderOAuthResolvedBatchImport {
access_token,
auth_config,
expires_at: result.token_set.expires_at_unix_secs,
});
}
if let Some(refresh_token) = refresh_token {
let Some(template) = template else {
if provider_type_supports_access_token_import(provider_type) {
@@ -228,6 +298,28 @@ pub(super) async fn execute_admin_provider_oauth_batch_import(
};
let template = admin_provider_oauth_template(provider_type);
if template.is_none()
&& !provider_type.eq_ignore_ascii_case("windsurf")
&& !provider_type_supports_access_token_import(provider_type)
{
return Ok(AdminProviderOAuthBatchImportOutcome {
total: entries.len(),
success: 0,
failed: entries.len(),
results: entries
.iter()
.enumerate()
.map(|(index, _)| {
json!({
"index": index,
"status": "error",
"error": ADMIN_PROVIDER_OAUTH_DATA_UNAVAILABLE_DETAIL,
"replaced": false,
})
})
.collect(),
});
}
let endpoint_resolution =
resolve_provider_oauth_runtime_endpoints(state, &provider, provider_type).await?;
@@ -251,6 +343,25 @@ pub(super) async fn execute_admin_provider_oauth_batch_import(
let mut failed = 0usize;
for (index, entry) in entries.iter().enumerate() {
if let Some(error) = entry.parse_error.as_ref() {
failed += 1;
results.push(json!({
"index": index,
"status": "error",
"error": error,
"replaced": false,
}));
maybe_report_admin_provider_oauth_batch_import_progress(
&mut progress,
entries.len(),
success,
failed,
&results,
)
.await;
continue;
}
let resolved_import = match resolve_admin_provider_oauth_batch_import_tokens(
state,
template,
@@ -418,3 +529,33 @@ pub(super) async fn execute_admin_provider_oauth_batch_import(
results,
})
}
#[cfg(test)]
mod tests {
use super::sanitize_windsurf_batch_import_error;
use aether_oauth::core::OAuthError;
#[test]
fn windsurf_batch_import_error_redacts_http_body() {
let error = OAuthError::HttpStatus {
status_code: 401,
body_excerpt: "sessionToken=devin-session-token$secret".to_string(),
};
let detail = sanitize_windsurf_batch_import_error(&error);
assert_eq!(detail, "Windsurf 凭据验证失败: HTTP 401");
assert!(!detail.contains("devin-session-token$secret"));
}
#[test]
fn windsurf_batch_import_error_redacts_provider_detail() {
let error = OAuthError::invalid_response("apiKey=sk-secret token=secret-token");
let detail = sanitize_windsurf_batch_import_error(&error);
assert_eq!(detail, "Windsurf 凭据验证失败");
assert!(!detail.contains("sk-secret"));
assert!(!detail.contains("secret-token"));
}
}

View File

@@ -8,7 +8,7 @@ use super::parse::{
};
use crate::handlers::admin::provider::oauth::errors::build_internal_control_error_response;
use crate::handlers::admin::provider::oauth::state::{
build_admin_provider_oauth_backend_unavailable_response,
admin_provider_oauth_template, build_admin_provider_oauth_backend_unavailable_response,
is_fixed_provider_type_for_provider_oauth,
};
use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_batch_import_provider_id;
@@ -60,6 +60,13 @@ pub(in super::super) async fn handle_admin_provider_oauth_batch_import(
"该 Provider 不是固定类型,无法使用 provider-oauth",
));
}
if provider_type != "kiro"
&& provider_type != "windsurf"
&& admin_provider_oauth_template(&provider_type).is_none()
{
return Ok(build_admin_provider_oauth_backend_unavailable_response());
}
let total = estimate_admin_provider_oauth_batch_import_total(
&provider_type,
payload.credentials.as_str(),

View File

@@ -19,8 +19,10 @@ pub(super) struct AdminProviderOAuthBatchImportRequest {
#[derive(Debug, Clone)]
pub(super) struct AdminProviderOAuthBatchImportEntry {
pub parse_error: Option<String>,
pub refresh_token: Option<String>,
pub access_token: Option<String>,
pub raw_credentials: Option<serde_json::Value>,
pub expires_at: Option<u64>,
pub account_id: Option<String>,
pub account_user_id: Option<String>,
@@ -141,8 +143,10 @@ fn extract_admin_provider_oauth_batch_import_entry(
access_token.as_deref(),
);
Some(AdminProviderOAuthBatchImportEntry {
parse_error: None,
refresh_token,
access_token,
raw_credentials: None,
expires_at: None,
account_id: None,
account_user_id: None,
@@ -160,6 +164,8 @@ fn extract_admin_provider_oauth_batch_import_entry(
}
}
serde_json::Value::Object(object) => {
let is_grok = provider_type.trim().eq_ignore_ascii_case("grok");
let is_windsurf = provider_type.trim().eq_ignore_ascii_case("windsurf");
let refresh_token = coerce_admin_provider_oauth_import_str(
object
.get("refresh_token")
@@ -170,12 +176,8 @@ fn extract_admin_provider_oauth_batch_import_entry(
.get("access_token")
.or_else(|| object.get("accessToken")),
);
let grok_token_alias = if provider_type.trim().eq_ignore_ascii_case("grok") {
object.get("token")
} else {
None
};
let grok_cookie = if provider_type.trim().eq_ignore_ascii_case("grok") {
let grok_token_alias = if is_grok { object.get("token") } else { None };
let grok_cookie = if is_grok {
coerce_admin_provider_oauth_import_str(
object.get("cookie").or_else(|| object.get("cookieHeader")),
)
@@ -198,9 +200,43 @@ fn extract_admin_provider_oauth_batch_import_entry(
refresh_token.as_deref(),
access_token.as_deref().or(session_token.as_deref()),
);
if refresh_token.is_none() && access_token.is_none() {
let windsurf_api_key = is_windsurf
.then(|| {
coerce_admin_provider_oauth_import_str(
object.get("api_key").or_else(|| object.get("apiKey")),
)
})
.flatten();
let windsurf_token = is_windsurf
.then(|| {
coerce_admin_provider_oauth_import_str(
object
.get("token")
.or_else(|| object.get("auth_token"))
.or_else(|| object.get("authToken")),
)
})
.flatten();
let windsurf_password = is_windsurf
.then(|| coerce_admin_provider_oauth_import_str(object.get("password")))
.flatten();
let raw_credentials = if is_windsurf
&& (windsurf_api_key.is_some()
|| windsurf_token.is_some()
|| windsurf_password.is_some())
{
Some(item.clone())
} else {
None
};
if refresh_token.is_none() && access_token.is_none() && raw_credentials.is_none() {
return None;
}
let refresh_token = if is_windsurf {
refresh_token.or(windsurf_api_key).or(windsurf_token)
} else {
refresh_token
};
let expires_at =
json_u64_value(object.get("expires_at").or_else(|| object.get("expiresAt")));
let account_id = coerce_admin_provider_oauth_import_str(
@@ -285,8 +321,10 @@ fn extract_admin_provider_oauth_batch_import_entry(
.or_else(|| object.get("impersonate")),
);
Some(AdminProviderOAuthBatchImportEntry {
parse_error: None,
refresh_token,
access_token,
raw_credentials,
expires_at,
account_id,
account_user_id,
@@ -316,14 +354,17 @@ pub(super) fn parse_admin_provider_oauth_batch_import_entries(
}
if raw.starts_with('[') {
if let Ok(serde_json::Value::Array(items)) = serde_json::from_str::<serde_json::Value>(raw)
{
return items
.iter()
.filter_map(|item| {
extract_admin_provider_oauth_batch_import_entry(provider_type, item)
})
.collect();
match serde_json::from_str::<serde_json::Value>(raw) {
Ok(serde_json::Value::Array(items)) => {
return items
.iter()
.filter_map(|item| {
extract_admin_provider_oauth_batch_import_entry(provider_type, item)
})
.collect();
}
Ok(_) => {}
Err(error) => return vec![parse_error_entry(format!("JSON 数组解析失败: {error}"))],
}
}
@@ -340,23 +381,61 @@ pub(super) fn parse_admin_provider_oauth_batch_import_entries(
raw.lines()
.map(str::trim)
.filter(|line| !line.is_empty() && !line.starts_with('#'))
.filter_map(|line| {
if line.starts_with('{') {
return serde_json::from_str::<serde_json::Value>(line)
.ok()
.and_then(|value| {
extract_admin_provider_oauth_batch_import_entry(provider_type, &value)
});
.filter_map(|token| {
if is_json_like_batch_line(token) {
match serde_json::from_str::<serde_json::Value>(token) {
Ok(value @ serde_json::Value::Object(_)) => {
return extract_admin_provider_oauth_batch_import_entry(
provider_type,
&value,
);
}
Ok(_) => {
return Some(parse_error_entry(
"JSON 行必须是账号对象,不能作为 raw token 导入".to_string(),
));
}
Err(error) => {
return Some(parse_error_entry(format!("JSON 行解析失败: {error}")));
}
}
}
extract_admin_provider_oauth_batch_import_entry(
provider_type,
&serde_json::Value::String(line.to_string()),
&serde_json::Value::String(token.to_string()),
)
})
.collect()
}
fn parse_error_entry(error: String) -> AdminProviderOAuthBatchImportEntry {
AdminProviderOAuthBatchImportEntry {
parse_error: Some(error),
refresh_token: None,
access_token: None,
raw_credentials: None,
expires_at: None,
account_id: None,
account_user_id: None,
plan_type: None,
pool_tier: None,
user_id: None,
email: None,
account_name: None,
sso_rw_token: None,
cf_cookies: None,
cf_clearance: None,
user_agent: None,
browser_profile: None,
}
}
fn is_json_like_batch_line(line: &str) -> bool {
let line = line.trim_start();
line.starts_with('{') || line.starts_with('[')
}
pub(super) fn apply_admin_provider_oauth_batch_import_hints(
provider_type: &str,
entry: &AdminProviderOAuthBatchImportEntry,
@@ -633,4 +712,115 @@ mod tests {
assert_eq!(entries[0].user_id.as_deref(), Some("user-1"));
assert_eq!(entries[0].pool_tier.as_deref(), Some("heavy"));
}
#[test]
fn parses_windsurf_json_credentials_for_native_import() {
let entries = parse_admin_provider_oauth_batch_import_entries(
"windsurf",
r#"[
{"api_key":"devin-session-token$abc","email":"a@example.com"},
{"token":"firebase-id-token","name":"Browser Login"},
{"email":"b@example.com","password":"secret"},
{"access_token":"devin-session-token$alias","email":"c@example.com"}
]"#,
);
assert_eq!(entries.len(), 4);
assert_eq!(
entries[0].refresh_token.as_deref(),
Some("devin-session-token$abc")
);
assert_eq!(entries[0].email.as_deref(), Some("a@example.com"));
assert_eq!(
entries[0]
.raw_credentials
.as_ref()
.and_then(|value| value.get("api_key")),
Some(&json!("devin-session-token$abc"))
);
assert_eq!(
entries[1]
.raw_credentials
.as_ref()
.and_then(|value| value.get("token")),
Some(&json!("firebase-id-token"))
);
assert_eq!(
entries[2]
.raw_credentials
.as_ref()
.and_then(|value| value.get("password")),
Some(&json!("secret"))
);
assert_eq!(
entries[3].access_token.as_deref(),
Some("devin-session-token$alias")
);
}
#[test]
fn parses_windsurf_json_lines_credentials_for_native_import() {
let entries = parse_admin_provider_oauth_batch_import_entries(
"windsurf",
r#"{"api_key":"devin-session-token$abc","email":"a@example.com"}
{"token":"firebase-id-token","name":"Browser Login"}
{"email":"b@example.com","password":"secret"}"#,
);
assert_eq!(entries.len(), 3);
assert_eq!(
entries[0]
.raw_credentials
.as_ref()
.and_then(|value| value.get("api_key")),
Some(&json!("devin-session-token$abc"))
);
assert_eq!(
entries[1]
.raw_credentials
.as_ref()
.and_then(|value| value.get("token")),
Some(&json!("firebase-id-token"))
);
assert_eq!(
entries[2]
.raw_credentials
.as_ref()
.and_then(|value| value.get("password")),
Some(&json!("secret"))
);
}
#[test]
fn invalid_json_line_is_parse_error_not_token() {
let entries = parse_admin_provider_oauth_batch_import_entries(
"windsurf",
r#"{"email":"b@example.com","password":"secret""#,
);
assert_eq!(entries.len(), 1);
assert!(entries[0].parse_error.is_some());
assert!(entries[0].refresh_token.is_none());
assert!(entries[0].access_token.is_none());
assert!(entries[0].raw_credentials.is_none());
}
#[test]
fn json_like_line_after_token_is_parse_error_not_token() {
let entries = parse_admin_provider_oauth_batch_import_entries(
"windsurf",
"devin-session-token$abc\n[not-json",
);
assert_eq!(entries.len(), 2);
assert!(entries[0].parse_error.is_none());
assert_eq!(
entries[0].refresh_token.as_deref(),
Some("devin-session-token$abc")
);
assert!(entries[1].parse_error.is_some());
assert!(entries[1].refresh_token.is_none());
assert!(entries[1].access_token.is_none());
assert!(entries[1].raw_credentials.is_none());
}
}

View File

@@ -10,7 +10,7 @@ use super::progress::{
};
use crate::handlers::admin::provider::oauth::errors::build_internal_control_error_response;
use crate::handlers::admin::provider::oauth::state::{
build_admin_provider_oauth_backend_unavailable_response,
admin_provider_oauth_template, build_admin_provider_oauth_backend_unavailable_response,
is_fixed_provider_type_for_provider_oauth,
};
use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_batch_import_task_provider_id;
@@ -124,6 +124,13 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
"该 Provider 不是固定类型,无法使用 provider-oauth",
));
}
if provider_type != "kiro"
&& provider_type != "windsurf"
&& admin_provider_oauth_template(&provider_type).is_none()
{
return Ok(build_admin_provider_oauth_backend_unavailable_response());
}
let total = estimate_admin_provider_oauth_batch_import_total(
&provider_type,
payload.credentials.as_str(),

View File

@@ -12,6 +12,7 @@ use crate::GatewayError;
use aether_data::repository::provider_oauth::{
StoredAdminProviderOAuthDeviceSession, KIRO_DEVICE_AUTH_SESSION_TTL_BUFFER_SECS,
};
use aether_oauth::provider::{ProviderOAuthService, ProviderOAuthTransportContext};
use axum::{
body::{Body, Bytes},
http,
@@ -28,6 +29,8 @@ const KIRO_SOCIAL_MANUAL_CALLBACK_PORT: u16 = 49153;
const KIRO_SOCIAL_ALLOWED_CALLBACK_PORTS: &[u16] = &[
3128, 4649, 6588, 8008, 9091, 49153, 50153, 51153, 52153, 53153,
];
const WINDSURF_BROWSER_AUTH_EXPIRES_IN_SECS: u64 = 600;
const WINDSURF_BROWSER_AUTH_POLL_INTERVAL_SECS: u64 = 5;
fn normalize_kiro_device_auth_type(raw: Option<&str>) -> String {
match raw
@@ -119,6 +122,26 @@ fn build_kiro_social_authorization_url(
)
}
fn build_windsurf_authorization_url(authorize_url: &str, login_option: &str) -> String {
let login_option = login_option.trim();
if login_option.is_empty() {
return authorize_url.to_string();
}
if let Ok(mut url) = Url::parse(authorize_url) {
url.query_pairs_mut()
.append_pair("login_option", login_option);
return url.to_string();
}
let separator = if authorize_url.contains('?') {
'&'
} else {
'?'
};
let mut serializer = form_urlencoded::Serializer::new(String::new());
serializer.append_pair("login_option", login_option);
format!("{authorize_url}{separator}{}", serializer.finish())
}
pub(super) async fn handle_admin_provider_oauth_device_authorize(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -163,14 +186,14 @@ pub(super) async fn handle_admin_provider_oauth_device_authorize(
));
};
let provider_type = provider.provider_type.trim().to_ascii_lowercase();
if provider_type != "kiro" {
if provider_type != "kiro" && provider_type != "windsurf" {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"设备授权仅支持 Kiro provider",
"设备授权仅支持 Kiro / Windsurf provider",
));
}
let endpoint_resolution =
resolve_provider_oauth_runtime_endpoints(state, &provider, "kiro").await?;
resolve_provider_oauth_runtime_endpoints(state, &provider, &provider_type).await?;
let runtime_endpoint = endpoint_resolution.runtime_endpoint;
let request_proxy = state
.resolve_admin_provider_oauth_operation_proxy_snapshot(
@@ -184,6 +207,103 @@ pub(super) async fn handle_admin_provider_oauth_device_authorize(
)
.await;
if provider_type == "windsurf" {
let session_id = generate_provider_oauth_nonce();
let login_option = payload
.login_option
.as_deref()
.or(payload.auth_type.as_deref())
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("default")
.to_ascii_lowercase();
let ctx = ProviderOAuthTransportContext {
provider_id: provider_id.clone(),
provider_type: provider_type.clone(),
endpoint_id: runtime_endpoint
.as_ref()
.map(|endpoint| endpoint.id.clone()),
key_id: None,
auth_type: Some("oauth".to_string()),
decrypted_api_key: None,
decrypted_auth_config: None,
provider_config: provider.config.clone(),
endpoint_config: runtime_endpoint
.as_ref()
.and_then(|endpoint| endpoint.config.clone()),
key_config: None,
network: aether_oauth::network::OAuthNetworkContext::provider_operation(
request_proxy.clone(),
),
};
let mut authorization = match ProviderOAuthService::with_builtin_adapters()
.build_authorize_url(&ctx, &session_id, None)
{
Ok(authorization) => authorization,
Err(error) => {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
format!("Windsurf 授权 URL 构建失败: {error}"),
));
}
};
authorization.authorize_url =
build_windsurf_authorization_url(&authorization.authorize_url, &login_option);
let now_unix_secs = current_unix_secs();
let session = StoredAdminProviderOAuthDeviceSession {
provider_id: provider_id.clone(),
region: String::new(),
client_id: String::new(),
client_secret: String::new(),
device_code: String::new(),
auth_type: Some("browser".to_string()),
social_provider: Some(login_option.clone()),
code_verifier: None,
redirect_uri: Some("show-auth-token".to_string()),
machine_id: Some(uuid::Uuid::new_v4().to_string().to_ascii_lowercase()),
interval: WINDSURF_BROWSER_AUTH_POLL_INTERVAL_SECS,
expires_at_unix_secs: now_unix_secs
.saturating_add(WINDSURF_BROWSER_AUTH_EXPIRES_IN_SECS),
status: "pending".to_string(),
proxy_node_id: payload
.proxy_node_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
created_at_unix_ms: now_unix_secs,
key_id: None,
email: None,
replaced: false,
error_msg: None,
};
if let Err(response) = state
.save_provider_oauth_device_session(
&session_id,
&session,
WINDSURF_BROWSER_AUTH_EXPIRES_IN_SECS
.saturating_add(KIRO_DEVICE_AUTH_SESSION_TTL_BUFFER_SECS),
)
.await
{
return Ok(response);
}
return Ok(Json(json!({
"session_id": session_id,
"user_code": "",
"verification_uri": "https://windsurf.com/windsurf/signin",
"verification_uri_complete": authorization.authorize_url,
"expires_in": WINDSURF_BROWSER_AUTH_EXPIRES_IN_SECS,
"interval": WINDSURF_BROWSER_AUTH_POLL_INTERVAL_SECS,
"auth_type": "browser",
"login_option": login_option,
"redirect_uri": "show-auth-token",
"callback_required": true,
}))
.into_response());
}
let auth_type = normalize_kiro_device_auth_type(payload.auth_type.as_deref());
if let Some(social_provider) = kiro_social_provider_id(&auth_type) {
let redirect_uri = match normalize_kiro_social_redirect_uri(payload.redirect_uri.as_deref())
@@ -394,3 +514,27 @@ pub(super) async fn handle_admin_provider_oauth_device_authorize(
}))
.into_response())
}
#[cfg(test)]
mod tests {
use super::build_windsurf_authorization_url;
#[test]
fn windsurf_authorization_url_includes_login_option() {
let url = build_windsurf_authorization_url(
"https://windsurf.com/windsurf/signin?state=session-1",
"github",
);
let parsed = url::Url::parse(&url).expect("url should parse");
let params = parsed
.query_pairs()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(params.get("state").map(String::as_str), Some("session-1"));
assert_eq!(
params.get("login_option").map(String::as_str),
Some("github")
);
}
}

View File

@@ -26,6 +26,9 @@ use aether_data::repository::provider_oauth::StoredAdminProviderOAuthDeviceSessi
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
};
use aether_oauth::provider::{
ProviderOAuthImportInput, ProviderOAuthService, ProviderOAuthTransportContext,
};
use axum::{
body::{Body, Bytes},
http,
@@ -86,6 +89,99 @@ fn kiro_social_poll_error_response(error: impl Into<String>) -> Response<Body> {
.into_response()
}
fn windsurf_browser_poll_error_response(error: impl Into<String>) -> Response<Body> {
Json(json!({
"status": "error",
"error": error.into(),
"replaced": false,
}))
.into_response()
}
fn sanitize_windsurf_browser_poll_detail(detail: impl AsRef<str>) -> String {
let detail = detail.as_ref().trim();
if detail.is_empty() {
return "-".to_string();
}
if contains_windsurf_sensitive_marker(detail) {
"[REDACTED upstream error body]".to_string()
} else {
detail.chars().take(500).collect()
}
}
fn sanitize_windsurf_browser_poll_callback_error(error: &str, description: &str) -> String {
let error = sanitize_windsurf_browser_poll_error_code(error);
let description = sanitize_windsurf_browser_poll_detail(description);
format!("{error}: {description}")
}
fn sanitize_windsurf_browser_poll_error_code(error: &str) -> String {
let error = error.trim();
if !error.is_empty()
&& error.len() <= 80
&& error
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
{
return error.to_string();
}
sanitize_windsurf_browser_poll_detail(error)
}
fn sanitize_windsurf_browser_poll_oauth_error(error: &aether_oauth::core::OAuthError) -> String {
match error {
aether_oauth::core::OAuthError::InvalidRequest(_) => {
"Windsurf token 验证失败: 请求参数无效".to_string()
}
aether_oauth::core::OAuthError::HttpStatus { status_code, .. } => {
format!("Windsurf token 验证失败: HTTP {status_code}")
}
_ => "Windsurf token 验证失败".to_string(),
}
}
fn contains_windsurf_sensitive_marker(value: &str) -> bool {
let lowered = value.to_ascii_lowercase();
[
"token",
"api_key",
"apikey",
"sessiontoken",
"firebase_id_token",
"idtoken",
"authorization",
"password",
"secret",
"devin-session-token$",
]
.iter()
.any(|marker| lowered.contains(marker))
|| value.contains("sk-")
}
fn secret_fingerprint(value: &str) -> Option<String> {
let value = value.trim();
if value.is_empty() {
return None;
}
use sha2::{Digest, Sha256};
let digest = Sha256::digest(value.as_bytes());
Some(
digest[..8]
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>(),
)
}
fn insert_secret_fingerprint(target: &mut serde_json::Map<String, Value>, key: &str, secret: &str) {
if let Some(fingerprint) = secret_fingerprint(secret) {
target.insert(key.to_string(), json!(fingerprint));
}
}
fn kiro_social_provider_from_login_option(login_option: Option<&str>) -> Option<&'static str> {
match login_option
.map(str::trim)
@@ -322,8 +418,9 @@ pub(super) async fn handle_admin_provider_oauth_device_poll(
"Provider 不存在",
));
};
let provider_type = provider.provider_type.trim().to_ascii_lowercase();
let endpoint_resolution =
resolve_provider_oauth_runtime_endpoints(state, &provider, "kiro").await?;
resolve_provider_oauth_runtime_endpoints(state, &provider, &provider_type).await?;
let endpoints = endpoint_resolution.endpoints;
let runtime_endpoint = endpoint_resolution.runtime_endpoint;
let request_proxy = state
@@ -338,6 +435,20 @@ pub(super) async fn handle_admin_provider_oauth_device_poll(
)
.await;
if provider_type == "windsurf" {
return handle_admin_provider_oauth_windsurf_browser_device_poll(
state,
&provider,
&endpoints,
request_proxy,
session_id,
session,
payload.callback_url.as_deref(),
payload.token.as_deref(),
)
.await;
}
if kiro_device_session_is_social(&session) {
return handle_admin_provider_oauth_kiro_social_device_poll(
state,
@@ -630,6 +741,276 @@ pub(super) async fn handle_admin_provider_oauth_device_poll(
))
}
fn windsurf_raw_api_key(value: &str) -> Option<&str> {
let value = value.trim();
if value.starts_with("devin-session-token$") || value.starts_with("sk-") {
Some(value)
} else {
None
}
}
async fn handle_admin_provider_oauth_windsurf_browser_device_poll(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
endpoints: &[StoredProviderCatalogEndpoint],
request_proxy: Option<ProxySnapshot>,
session_id: &str,
mut session: StoredAdminProviderOAuthDeviceSession,
callback_url: Option<&str>,
token: Option<&str>,
) -> Result<Response<Body>, GatewayError> {
let callback_url = callback_url
.map(str::trim)
.filter(|value| !value.is_empty());
let token = token.map(str::trim).filter(|value| !value.is_empty());
if callback_url.is_none() && token.is_none() {
return Ok(Json(json!({"status": "pending", "replaced": false})).into_response());
}
let mut social_provider = session
.social_provider
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let imported_token = if let Some(callback_url) = callback_url {
let callback_params = parse_provider_oauth_callback_params(callback_url);
if let Some(error) = callback_params.get("error").map(String::as_str) {
let error_description = callback_params
.get("error_description")
.map(String::as_str)
.unwrap_or("用户拒绝授权");
let sanitized_error =
sanitize_windsurf_browser_poll_callback_error(error, error_description);
session.status = "error".to_string();
session.error_msg = Some(sanitized_error.clone());
let _ = state
.save_provider_oauth_device_session(session_id, &session, 30)
.await;
return Ok(attach_admin_provider_oauth_device_poll_terminal_response(
session_id,
"error",
windsurf_browser_poll_error_response(sanitized_error),
));
}
let Some(callback_state) = callback_params
.get("state")
.map(String::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(windsurf_browser_poll_error_response("回调 URL 缺少 state"));
};
if callback_state != session_id {
return Ok(windsurf_browser_poll_error_response(
"回调 state 与会话不匹配",
));
}
if let Some(provider) = callback_params
.get("provider")
.or_else(|| callback_params.get("login_option"))
.map(String::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
social_provider = Some(provider.to_string());
}
let Some(callback_token) = callback_params
.get("token")
.or_else(|| callback_params.get("auth_token"))
.or_else(|| callback_params.get("access_token"))
.map(String::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(windsurf_browser_poll_error_response("回调 URL 缺少 token"));
};
callback_token.to_string()
} else {
let token = token.unwrap_or_default();
if windsurf_raw_api_key(token).is_none() {
return Ok(windsurf_browser_poll_error_response(
"浏览器授权请提交包含 state 的回调 URL纯 token 请使用导入授权",
));
}
token.to_string()
};
let mut raw_credentials = serde_json::Map::new();
if windsurf_raw_api_key(&imported_token).is_some() {
raw_credentials.insert("api_key".to_string(), json!(imported_token));
} else {
raw_credentials.insert("token".to_string(), json!(imported_token));
}
if let Some(social_provider) = social_provider.as_ref() {
raw_credentials.insert("social_provider".to_string(), json!(social_provider));
}
let ctx = ProviderOAuthTransportContext {
provider_id: provider.id.clone(),
provider_type: provider.provider_type.clone(),
endpoint_id: None,
key_id: None,
auth_type: Some("oauth".to_string()),
decrypted_api_key: None,
decrypted_auth_config: None,
provider_config: provider.config.clone(),
endpoint_config: None,
key_config: None,
network: aether_oauth::network::OAuthNetworkContext::provider_operation(
request_proxy.clone(),
),
};
let executor = crate::oauth::GatewayOAuthHttpExecutor::new(*state);
let result = match ProviderOAuthService::with_builtin_adapters()
.import_credentials(
&executor,
&ctx,
ProviderOAuthImportInput {
provider_type: provider.provider_type.clone(),
name: None,
refresh_token: None,
raw_credentials: Some(Value::Object(raw_credentials)),
network: ctx.network.clone(),
},
)
.await
{
Ok(result) => result,
Err(error) => {
let sanitized_error = sanitize_windsurf_browser_poll_oauth_error(&error);
session.status = "error".to_string();
session.error_msg = Some(sanitized_error.clone());
let _ = state
.save_provider_oauth_device_session(session_id, &session, 30)
.await;
return Ok(attach_admin_provider_oauth_device_poll_terminal_response(
session_id,
"error",
windsurf_browser_poll_error_response(sanitized_error),
));
}
};
let access_token = result.token_set.access_token.trim().to_string();
if access_token.is_empty() {
return Ok(windsurf_browser_poll_error_response(
"Windsurf token 验证返回缺少 apiKey/sessionToken",
));
}
let mut auth_config = result.auth_config.as_object().cloned().unwrap_or_default();
auth_config.insert("provider_type".to_string(), json!("windsurf"));
auth_config.insert("auth_method".to_string(), json!("browser"));
if let Some(social_provider) = social_provider.as_ref() {
auth_config
.entry("social_provider".to_string())
.or_insert_with(|| json!(social_provider));
}
let duplicate = match state
.find_duplicate_provider_oauth_key(&provider.id, &auth_config, None)
.await
{
Ok(duplicate) => duplicate,
Err(detail) => {
return Ok(Json(json!({
"status": "error",
"error": detail,
"replaced": false,
}))
.into_response());
}
};
let api_formats = provider_oauth_active_api_formats(endpoints);
let key_proxy = provider_oauth_key_proxy_value(session.proxy_node_id.as_deref());
let expires_at = result.token_set.expires_at_unix_secs;
let email = auth_config
.get("email")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let mut replaced = false;
let persisted_key = if let Some(existing_key) = duplicate {
replaced = true;
match state
.update_existing_provider_oauth_catalog_key(
&existing_key,
&provider.provider_type,
&access_token,
&auth_config,
&api_formats,
key_proxy.clone(),
expires_at,
)
.await?
{
Some(key) => key,
None => {
return Ok(build_internal_control_error_response(
http::StatusCode::SERVICE_UNAVAILABLE,
"provider oauth write unavailable",
));
}
}
} else {
let key_name = email
.as_deref()
.map(|email| format!("windsurf_{email}"))
.unwrap_or_else(|| format!("windsurf_{}", current_unix_secs()));
match state
.create_provider_oauth_catalog_key(
&provider.id,
&provider.provider_type,
&key_name,
&access_token,
&auth_config,
&api_formats,
key_proxy,
expires_at,
)
.await?
{
Some(key) => key,
None => {
return Ok(build_internal_control_error_response(
http::StatusCode::SERVICE_UNAVAILABLE,
"provider oauth write unavailable",
));
}
}
};
spawn_provider_oauth_account_state_refresh_after_update(
state.cloned_app(),
provider.clone(),
persisted_key.id.clone(),
request_proxy.clone(),
);
session.status = "authorized".to_string();
session.key_id = Some(persisted_key.id.clone());
session.email = email.clone();
session.replaced = replaced;
session.error_msg = None;
let _ = state
.save_provider_oauth_device_session(session_id, &session, 60)
.await;
Ok(attach_admin_provider_oauth_device_poll_terminal_response(
session_id,
"authorized",
Json(json!({
"status": "authorized",
"key_id": persisted_key.id,
"email": email,
"replaced": replaced,
}))
.into_response(),
))
}
async fn handle_admin_provider_oauth_kiro_social_device_poll(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
@@ -814,7 +1195,7 @@ async fn handle_admin_provider_oauth_kiro_social_device_poll(
.get("idToken")
.or_else(|| token_result.get("id_token")),
) {
auth_config_object.insert("id_token".to_string(), json!(id_token));
insert_secret_fingerprint(&mut auth_config_object, "id_token_fingerprint", &id_token);
}
if let Some(token_type) = json_non_empty_string(
token_result
@@ -921,3 +1302,18 @@ async fn handle_admin_provider_oauth_kiro_social_device_poll(
.into_response(),
))
}
#[cfg(test)]
mod tests {
#[test]
fn windsurf_browser_poll_callback_error_redacts_sensitive_values() {
let detail = super::sanitize_windsurf_browser_poll_callback_error(
"access_denied",
"bad token devin-session-token$secret and apiKey sk-secret",
);
assert_eq!(detail, "access_denied: [REDACTED upstream error body]");
assert!(!detail.contains("devin-session-token$secret"));
assert!(!detail.contains("sk-secret"));
}
}

View File

@@ -12,6 +12,7 @@ pub(super) struct AdminProviderOAuthDeviceAuthorizePayload {
#[serde(default = "default_kiro_device_region")]
pub(super) region: String,
pub(super) auth_type: Option<String>,
pub(super) login_option: Option<String>,
pub(super) redirect_uri: Option<String>,
pub(super) proxy_node_id: Option<String>,
}
@@ -20,6 +21,7 @@ pub(super) struct AdminProviderOAuthDeviceAuthorizePayload {
pub(super) struct AdminProviderOAuthDevicePollPayload {
pub(super) session_id: String,
pub(super) callback_url: Option<String>,
pub(super) token: Option<String>,
}
pub(super) fn attach_admin_provider_oauth_device_poll_terminal_response(

View File

@@ -25,6 +25,10 @@ use crate::handlers::admin::request::{
};
use crate::GatewayError;
use aether_contracts::ProxySnapshot;
use aether_oauth::core::OAuthError;
use aether_oauth::provider::{
ProviderOAuthImportInput, ProviderOAuthService, ProviderOAuthTransportContext,
};
use axum::{
body::Body,
http,
@@ -39,6 +43,49 @@ struct AdminProviderOAuthSingleImportTokens {
expires_at: Option<u64>,
}
fn sanitize_windsurf_import_error(error: &OAuthError) -> String {
match error {
OAuthError::InvalidRequest(_) => "Windsurf 凭据验证失败: 请求参数无效".to_string(),
OAuthError::HttpStatus { status_code, .. } => {
format!("Windsurf 凭据验证失败: HTTP {status_code}")
}
OAuthError::InvalidResponse(detail) => sanitize_windsurf_invalid_response_detail(detail)
.unwrap_or_else(|| "Windsurf 凭据验证失败".to_string()),
_ => "Windsurf 凭据验证失败".to_string(),
}
}
fn sanitize_windsurf_invalid_response_detail(detail: &str) -> Option<String> {
let detail = detail.trim();
if detail.eq_ignore_ascii_case("Auth1 response is not json") {
return Some("Windsurf 凭据验证失败: Auth1 响应无法解析".to_string());
}
if detail.eq_ignore_ascii_case("Auth1 response missing token") {
return Some("Windsurf 凭据验证失败: Auth1 响应缺少 token".to_string());
}
if detail.contains("WindsurfPostAuth response missing sessionToken")
|| detail.contains("WindsurfPostAuth response is not json")
|| (detail.contains("WindsurfPostAuth failed") && detail.contains("missing sessionToken"))
{
return Some("Windsurf 凭据验证失败: PostAuth 未返回 sessionToken".to_string());
}
if detail.contains("WindsurfPostAuth failed") {
return Some("Windsurf 凭据验证失败: PostAuth 失败".to_string());
}
None
}
fn import_payload_has_windsurf_credentials(
payload: &serde_json::Map<String, serde_json::Value>,
) -> bool {
import_payload_string(payload, "api_key", "apiKey").is_some()
|| import_payload_string_any(payload, &["token", "auth_token", "authToken"]).is_some()
|| import_payload_string(payload, "refresh_token", "refreshToken").is_some()
|| import_payload_string(payload, "access_token", "accessToken").is_some()
|| (import_payload_string_any(payload, &["email"]).is_some()
&& import_payload_string_any(payload, &["password"]).is_some())
}
fn import_payload_string(
payload: &serde_json::Map<String, serde_json::Value>,
snake_case: &str,
@@ -266,6 +313,70 @@ async fn resolve_admin_provider_oauth_single_import_tokens(
})
}
async fn resolve_admin_provider_oauth_windsurf_single_import_tokens(
state: &AdminAppState<'_>,
provider_type: &str,
name: Option<String>,
raw_payload: &serde_json::Map<String, serde_json::Value>,
refresh_token: Option<&str>,
request_proxy: Option<ProxySnapshot>,
) -> Result<AdminProviderOAuthSingleImportTokens, Response<Body>> {
let ctx = ProviderOAuthTransportContext {
provider_id: String::new(),
provider_type: provider_type.to_string(),
endpoint_id: None,
key_id: None,
auth_type: Some("oauth".to_string()),
decrypted_api_key: None,
decrypted_auth_config: None,
provider_config: None,
endpoint_config: None,
key_config: None,
network: aether_oauth::network::OAuthNetworkContext::provider_operation(
request_proxy.clone(),
),
};
let executor = crate::oauth::GatewayOAuthHttpExecutor::new(*state);
let service = ProviderOAuthService::with_builtin_adapters();
let result = service
.import_credentials(
&executor,
&ctx,
ProviderOAuthImportInput {
provider_type: provider_type.to_string(),
name,
refresh_token: refresh_token.map(ToOwned::to_owned),
raw_credentials: Some(serde_json::Value::Object(raw_payload.clone())),
network: ctx.network.clone(),
},
)
.await
.map_err(|error| {
build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
sanitize_windsurf_import_error(&error),
)
})?;
let access_token = result.token_set.access_token.trim().to_string();
if access_token.is_empty() {
return Err(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Windsurf 凭据验证返回缺少 apiKey/sessionToken",
));
}
let auth_config = result.auth_config.as_object().cloned().ok_or_else(|| {
build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Windsurf 凭据验证返回缺少 auth_config",
)
})?;
Ok(AdminProviderOAuthSingleImportTokens {
access_token,
auth_config,
expires_at: result.token_set.expires_at_unix_secs,
})
}
pub(super) async fn handle_admin_provider_oauth_import_refresh_token(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -370,19 +481,47 @@ pub(super) async fn handle_admin_provider_oauth_import_refresh_token(
.await;
let key_proxy = provider_oauth_key_proxy_value(proxy_node_id.as_deref());
let resolved_import = match resolve_admin_provider_oauth_single_import_tokens(
state,
template,
&provider_type,
refresh_token_input.as_deref(),
access_token_input.as_deref(),
imported_expires_at,
request_proxy.clone(),
)
.await
{
Ok(value) => value,
Err(response) => return Ok(response),
let resolved_import = if provider_type == "windsurf" {
if !import_payload_has_windsurf_credentials(&raw_payload) {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Windsurf 凭据不能为空",
));
}
match resolve_admin_provider_oauth_windsurf_single_import_tokens(
state,
&provider_type,
name.clone(),
&raw_payload,
refresh_token_input.as_deref(),
request_proxy.clone(),
)
.await
{
Ok(value) => value,
Err(response) => return Ok(response),
}
} else {
if refresh_token_input.is_none() && access_token_input.is_none() {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Refresh Token 或 Access Token 不能为空",
));
}
match resolve_admin_provider_oauth_single_import_tokens(
state,
template,
&provider_type,
refresh_token_input.as_deref(),
access_token_input.as_deref(),
imported_expires_at,
request_proxy.clone(),
)
.await
{
Ok(value) => value,
Err(response) => return Ok(response),
}
};
let AdminProviderOAuthSingleImportTokens {
access_token,
@@ -480,3 +619,47 @@ pub(super) async fn handle_admin_provider_oauth_import_refresh_token(
}))
.into_response())
}
#[cfg(test)]
mod tests {
use super::sanitize_windsurf_import_error;
use aether_oauth::core::OAuthError;
#[test]
fn windsurf_import_error_redacts_http_body() {
let error = OAuthError::HttpStatus {
status_code: 400,
body_excerpt: "token=secret-token password=secret-password".to_string(),
};
let detail = sanitize_windsurf_import_error(&error);
assert_eq!(detail, "Windsurf 凭据验证失败: HTTP 400");
assert!(!detail.contains("secret-token"));
assert!(!detail.contains("secret-password"));
}
#[test]
fn windsurf_import_error_redacts_invalid_response_detail() {
let error =
OAuthError::invalid_response("RegisterUser failed with firebase_id_token=secret-token");
let detail = sanitize_windsurf_import_error(&error);
assert_eq!(detail, "Windsurf 凭据验证失败");
assert!(!detail.contains("secret-token"));
assert!(!detail.contains("firebase_id_token"));
}
#[test]
fn windsurf_import_error_keeps_safe_post_auth_stage() {
let error = OAuthError::invalid_response("WindsurfPostAuth response missing sessionToken");
let detail = sanitize_windsurf_import_error(&error);
assert_eq!(
detail,
"Windsurf 凭据验证失败: PostAuth 未返回 sessionToken"
);
}
}

View File

@@ -63,6 +63,12 @@ pub(super) async fn handle_admin_provider_oauth_start_key(
"该 Provider 不是固定类型,无法使用 provider-oauth",
));
}
if provider_type == "windsurf" {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Windsurf 请使用浏览器登录或导入凭据。",
));
}
let Some(template) = admin_provider_oauth_template(&provider_type) else {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
@@ -134,6 +140,12 @@ pub(super) async fn handle_admin_provider_oauth_start_provider(
"Kiro 不支持 OAuth 授权,请使用导入授权。",
));
}
if provider_type == "windsurf" {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Windsurf 请使用浏览器登录或导入凭据。",
));
}
let Some(template) = admin_provider_oauth_template(&provider_type) else {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,

View File

@@ -36,6 +36,13 @@ fn is_openai_provider_oauth_provider_type(value: Option<&serde_json::Value>) ->
})
}
fn is_windsurf_provider_oauth_provider_type(value: Option<&serde_json::Value>) -> bool {
value
.and_then(serde_json::Value::as_str)
.map(str::trim)
.is_some_and(|provider_type| provider_type.eq_ignore_ascii_case("windsurf"))
}
fn match_codex_provider_oauth_identity(
new_auth_config: &serde_json::Map<String, serde_json::Value>,
existing_auth_config: &serde_json::Map<String, serde_json::Value>,
@@ -106,6 +113,41 @@ fn match_codex_provider_oauth_identity(
None
}
fn match_windsurf_provider_oauth_identity(
new_auth_config: &serde_json::Map<String, serde_json::Value>,
existing_auth_config: &serde_json::Map<String, serde_json::Value>,
) -> Option<bool> {
let new_provider_type = new_auth_config.get("provider_type");
let existing_provider_type = existing_auth_config.get("provider_type");
if !is_windsurf_provider_oauth_provider_type(new_provider_type)
&& !is_windsurf_provider_oauth_provider_type(existing_provider_type)
{
return None;
}
let new_account_id = normalize_provider_oauth_identity_value(new_auth_config.get("account_id"));
let existing_account_id =
normalize_provider_oauth_identity_value(existing_auth_config.get("account_id"));
if let (Some(new_account_id), Some(existing_account_id)) =
(new_account_id.as_deref(), existing_account_id.as_deref())
{
return Some(new_account_id == existing_account_id);
}
let new_credential_fingerprint =
normalize_provider_oauth_identity_value(new_auth_config.get("credential_fingerprint"));
let existing_credential_fingerprint =
normalize_provider_oauth_identity_value(existing_auth_config.get("credential_fingerprint"));
if let (Some(new_fingerprint), Some(existing_fingerprint)) = (
new_credential_fingerprint.as_deref(),
existing_credential_fingerprint.as_deref(),
) {
return Some(new_fingerprint == existing_fingerprint);
}
None
}
fn is_codex_cross_plan_group_non_duplicate(
new_auth_config: &serde_json::Map<String, serde_json::Value>,
existing_auth_config: &serde_json::Map<String, serde_json::Value>,
@@ -169,10 +211,17 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
) -> Result<Option<StoredProviderCatalogKey>, String> {
let new_email = normalize_provider_oauth_identity_value(auth_config.get("email"));
let new_user_id = normalize_provider_oauth_identity_value(auth_config.get("user_id"));
let new_account_id = normalize_provider_oauth_identity_value(auth_config.get("account_id"));
let new_credential_fingerprint =
normalize_provider_oauth_identity_value(auth_config.get("credential_fingerprint"));
let new_auth_method = normalize_provider_oauth_identity_value(auth_config.get("auth_method"));
let new_kiro_provider = normalize_provider_oauth_identity_value(auth_config.get("provider"));
if new_email.is_none() && new_user_id.is_none() {
if new_email.is_none()
&& new_user_id.is_none()
&& new_account_id.is_none()
&& new_credential_fingerprint.is_none()
{
return Ok(None);
}
@@ -201,15 +250,28 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
normalize_provider_oauth_identity_value(existing_auth_config.get("auth_method"));
let existing_kiro_provider =
normalize_provider_oauth_identity_value(existing_auth_config.get("provider"));
let is_windsurf = auth_config
.get("provider_type")
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.eq_ignore_ascii_case("windsurf"))
|| existing_auth_config
.get("provider_type")
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.eq_ignore_ascii_case("windsurf"));
let mut is_duplicate = false;
let codex_identity_match =
match_codex_provider_oauth_identity(auth_config, &existing_auth_config);
let windsurf_identity_match =
match_windsurf_provider_oauth_identity(auth_config, &existing_auth_config);
if let Some(codex_identity_match) = codex_identity_match {
is_duplicate = codex_identity_match;
} else if let Some(windsurf_identity_match) = windsurf_identity_match {
is_duplicate = windsurf_identity_match;
}
if codex_identity_match.is_none()
&& windsurf_identity_match.is_none()
&& !is_duplicate
&& new_user_id.is_some()
&& existing_user_id.is_some()
@@ -220,7 +282,9 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
}
if codex_identity_match.is_none()
&& windsurf_identity_match.is_none()
&& !is_duplicate
&& !is_windsurf
&& new_email.is_some()
&& existing_email.is_some()
&& new_email == existing_email
@@ -261,6 +325,12 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
let identifier =
normalize_provider_oauth_identity_value(auth_config.get("account_user_id"))
.or_else(|| normalize_provider_oauth_identity_value(auth_config.get("account_id")))
.or_else(|| {
normalize_provider_oauth_identity_value(
auth_config.get("credential_fingerprint"),
)
.map(|value| format!("fingerprint:{value}"))
})
.or_else(|| new_email.clone())
.or_else(|| new_user_id.clone())
.unwrap_or_default();
@@ -272,3 +342,91 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
Ok(None)
}
#[cfg(test)]
mod tests {
use super::match_windsurf_provider_oauth_identity;
use serde_json::{json, Map, Value};
fn auth_config(value: Value) -> Map<String, Value> {
value.as_object().cloned().expect("auth config object")
}
#[test]
fn windsurf_identity_matches_account_id_without_email() {
let new_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "api_key",
"account_id": "acct-ws-1"
}));
let existing_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "browser",
"account_id": "acct-ws-1"
}));
assert_eq!(
match_windsurf_provider_oauth_identity(&new_auth_config, &existing_auth_config),
Some(true)
);
}
#[test]
fn windsurf_identity_rejects_different_account_id() {
let new_auth_config = auth_config(json!({
"provider_type": "windsurf",
"account_id": "acct-ws-1",
"email": "same@example.com"
}));
let existing_auth_config = auth_config(json!({
"provider_type": "windsurf",
"account_id": "acct-ws-2",
"email": "same@example.com"
}));
assert_eq!(
match_windsurf_provider_oauth_identity(&new_auth_config, &existing_auth_config),
Some(false)
);
}
#[test]
fn windsurf_identity_matches_credential_fingerprint_without_profile() {
let new_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "api_key",
"credential_fingerprint": "abcdef0123456789"
}));
let existing_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "browser",
"credential_fingerprint": "abcdef0123456789"
}));
assert_eq!(
match_windsurf_provider_oauth_identity(&new_auth_config, &existing_auth_config),
Some(true)
);
}
#[test]
fn windsurf_identity_does_not_match_user_supplied_email_only() {
let new_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "api_key",
"email": "same@example.com",
"email_verified": false
}));
let existing_auth_config = auth_config(json!({
"provider_type": "windsurf",
"auth_method": "api_key",
"email": "same@example.com",
"email_verified": false
}));
assert_eq!(
match_windsurf_provider_oauth_identity(&new_auth_config, &existing_auth_config),
None
);
}
}

View File

@@ -6,6 +6,7 @@ use super::chatgpt_web::refresh_chatgpt_web_provider_quota_locally;
use super::codex::refresh_codex_provider_quota_locally;
use super::grok::refresh_grok_provider_quota_locally;
use super::kiro::refresh_kiro_provider_quota_locally;
use super::windsurf::refresh_windsurf_provider_quota_locally;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_contracts::ProxySnapshot;
@@ -36,6 +37,7 @@ const PROVIDER_QUOTA_REFRESH_HANDLERS: &[(&str, ProviderQuotaRefreshHandler)] =
("codex", refresh_codex_provider_quota_locally_boxed),
("grok", refresh_grok_provider_quota_locally_boxed),
("kiro", refresh_kiro_provider_quota_locally_boxed),
("windsurf", refresh_windsurf_provider_quota_locally_boxed),
];
pub(crate) async fn refresh_provider_pool_quota_locally(
@@ -135,3 +137,19 @@ fn refresh_grok_provider_quota_locally_boxed<'a>(
proxy_override,
))
}
fn refresh_windsurf_provider_quota_locally_boxed<'a>(
state: &'a AdminAppState<'a>,
provider: &'a StoredProviderCatalogProvider,
endpoint: &'a StoredProviderCatalogEndpoint,
keys: Vec<StoredProviderCatalogKey>,
proxy_override: Option<ProxySnapshot>,
) -> ProviderQuotaRefreshFuture<'a> {
Box::pin(refresh_windsurf_provider_quota_locally(
state,
provider,
endpoint,
keys,
proxy_override,
))
}

View File

@@ -5,3 +5,4 @@ pub(crate) mod dispatch;
pub(crate) mod grok;
pub(crate) mod kiro;
pub(crate) mod shared;
pub(crate) mod windsurf;

View File

@@ -0,0 +1,632 @@
use super::shared::{
build_provider_quota_execution_plan, build_quota_snapshot_payload,
default_provider_quota_execution_timeouts, execute_provider_quota_plan,
extract_execution_error_message, persist_provider_quota_refresh_state,
quota_refresh_success_invalid_state, ProviderQuotaExecutionOutcome,
};
use crate::handlers::admin::request::{AdminAppState, AdminGatewayProviderTransportSnapshot};
use crate::GatewayError;
use aether_contracts::ProxySnapshot;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_provider_pool::{
build_windsurf_pool_model_configs_request_with_base_url,
build_windsurf_pool_quota_request_with_base_url,
build_windsurf_pool_rate_limit_request_with_base_url, ProviderPoolQuotaRequestSpec,
};
use serde_json::json;
use std::time::{SystemTime, UNIX_EPOCH};
async fn execute_windsurf_probe_plan(
state: &AdminAppState<'_>,
transport: &AdminGatewayProviderTransportSnapshot,
spec: ProviderPoolQuotaRequestSpec,
proxy_override: Option<&ProxySnapshot>,
quota_kind: &str,
) -> Result<ProviderQuotaExecutionOutcome, GatewayError> {
let proxy = match proxy_override {
Some(proxy) => Some(proxy.clone()),
None => {
state
.resolve_transport_proxy_snapshot_with_tunnel_affinity(transport)
.await
}
};
let timeouts = state
.resolve_transport_execution_timeouts(transport)
.or(Some(default_provider_quota_execution_timeouts(
proxy.as_ref(),
)));
let plan = build_provider_quota_execution_plan(
transport,
spec,
proxy,
state.resolve_transport_profile(transport),
timeouts,
);
execute_provider_quota_plan(state, transport, plan, quota_kind).await
}
async fn execute_windsurf_user_status_plan(
state: &AdminAppState<'_>,
transport: &AdminGatewayProviderTransportSnapshot,
api_key: &str,
proxy_override: Option<&ProxySnapshot>,
) -> Result<ProviderQuotaExecutionOutcome, GatewayError> {
let spec = build_windsurf_pool_quota_request_with_base_url(
&transport.key.id,
&transport.endpoint.base_url,
api_key,
);
execute_windsurf_probe_plan(
state,
transport,
spec,
proxy_override,
"windsurf:user_status",
)
.await
}
async fn execute_windsurf_model_configs_plan(
state: &AdminAppState<'_>,
transport: &AdminGatewayProviderTransportSnapshot,
api_key: &str,
proxy_override: Option<&ProxySnapshot>,
) -> Result<ProviderQuotaExecutionOutcome, GatewayError> {
let spec = build_windsurf_pool_model_configs_request_with_base_url(
&transport.key.id,
&transport.endpoint.base_url,
api_key,
);
execute_windsurf_probe_plan(
state,
transport,
spec,
proxy_override,
"windsurf:model_configs",
)
.await
}
async fn execute_windsurf_rate_limit_plan(
state: &AdminAppState<'_>,
transport: &AdminGatewayProviderTransportSnapshot,
api_key: &str,
proxy_override: Option<&ProxySnapshot>,
) -> Result<ProviderQuotaExecutionOutcome, GatewayError> {
let spec = build_windsurf_pool_rate_limit_request_with_base_url(
&transport.key.id,
&transport.endpoint.base_url,
api_key,
);
execute_windsurf_probe_plan(
state,
transport,
spec,
proxy_override,
"windsurf:rate_limit",
)
.await
}
fn merge_windsurf_probe_metadata(
mut user_status_metadata: serde_json::Value,
model_configs_metadata: Option<serde_json::Value>,
rate_limit_metadata: Option<serde_json::Value>,
) -> serde_json::Value {
let Some(target) = user_status_metadata.as_object_mut() else {
return user_status_metadata;
};
for metadata in [model_configs_metadata, rate_limit_metadata]
.into_iter()
.flatten()
{
if let Some(source) = metadata.as_object() {
for (key, value) in source {
target.insert(key.clone(), value.clone());
}
}
}
user_status_metadata
}
fn append_windsurf_probe_warning(metadata: &mut serde_json::Value, probe: &str, message: String) {
let Some(target) = metadata.as_object_mut() else {
return;
};
let warnings = target
.entry("probe_warnings".to_string())
.or_insert_with(|| serde_json::Value::Array(Vec::new()));
if let Some(items) = warnings.as_array_mut() {
items.push(json!({
"probe": probe,
"message": message,
}));
}
}
fn build_windsurf_metadata_update(
current_upstream_metadata: Option<&serde_json::Value>,
patch: serde_json::Value,
) -> serde_json::Value {
let Some(patch_object) = patch.as_object() else {
return json!({ "windsurf": patch });
};
let mut merged_bucket = current_upstream_metadata
.and_then(|value| value.get("windsurf"))
.and_then(serde_json::Value::as_object)
.cloned()
.unwrap_or_default();
for (key, value) in patch_object {
merged_bucket.insert(key.clone(), value.clone());
}
json!({ "windsurf": merged_bucket })
}
fn sanitize_windsurf_probe_detail(detail: impl AsRef<str>) -> String {
let detail = detail.as_ref().trim();
if detail.is_empty() {
return "-".to_string();
}
if let Ok(mut value) = serde_json::from_str::<serde_json::Value>(detail) {
redact_windsurf_sensitive_json(&mut value);
return value.to_string().chars().take(500).collect();
}
if contains_windsurf_sensitive_marker(detail) {
"[REDACTED upstream error body]".to_string()
} else {
detail.chars().take(500).collect()
}
}
fn redact_windsurf_sensitive_json(value: &mut serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
for (key, value) in object {
if is_windsurf_sensitive_key(key) {
*value = json!("[REDACTED]");
} else {
redact_windsurf_sensitive_json(value);
}
}
}
serde_json::Value::Array(items) => {
for item in items {
redact_windsurf_sensitive_json(item);
}
}
serde_json::Value::String(text) if looks_like_windsurf_secret(text) => {
*text = "[REDACTED]".to_string();
}
_ => {}
}
}
fn is_windsurf_sensitive_key(key: &str) -> bool {
let normalized = key
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.collect::<String>()
.to_ascii_lowercase();
normalized.contains("token")
|| normalized.contains("apikey")
|| normalized.contains("password")
|| normalized.contains("authorization")
|| normalized.contains("secret")
}
fn looks_like_windsurf_secret(value: &str) -> bool {
let value = value.trim();
value.starts_with("devin-session-token$")
|| value.starts_with("sk-")
|| (value.len() > 80 && value.split('.').count() == 3)
}
fn contains_windsurf_sensitive_marker(value: &str) -> bool {
let lowered = value.to_ascii_lowercase();
[
"token",
"api_key",
"apikey",
"sessiontoken",
"firebase_id_token",
"idtoken",
"authorization",
"password",
"secret",
"devin-session-token$",
]
.iter()
.any(|marker| lowered.contains(marker))
|| value.contains("sk-")
}
pub(crate) async fn refresh_windsurf_provider_quota_locally(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
endpoint: &StoredProviderCatalogEndpoint,
keys: Vec<StoredProviderCatalogKey>,
proxy_override: Option<ProxySnapshot>,
) -> Result<Option<serde_json::Value>, GatewayError> {
let mut results = Vec::new();
let mut success_count = 0usize;
let mut failed_count = 0usize;
for key in keys {
let transport = match state
.read_provider_transport_snapshot(&provider.id, &endpoint.id, &key.id)
.await?
{
Some(transport) => transport,
None => {
failed_count += 1;
results.push(json!({
"key_id": key.id,
"key_name": key.name,
"status": "error",
"message": "Provider transport snapshot unavailable",
}));
continue;
}
};
let api_key = transport.key.decrypted_api_key.trim();
if api_key.is_empty() {
failed_count += 1;
results.push(json!({
"key_id": key.id,
"key_name": key.name,
"status": "error",
"message": "缺少 Windsurf apiKey/sessionToken",
}));
continue;
}
let result = match execute_windsurf_user_status_plan(
state,
&transport,
api_key,
proxy_override.as_ref(),
)
.await?
{
ProviderQuotaExecutionOutcome::Response(result) => result,
ProviderQuotaExecutionOutcome::Failure(detail) => {
failed_count += 1;
let detail = sanitize_windsurf_probe_detail(detail);
results.push(json!({
"key_id": key.id,
"key_name": key.name,
"status": "error",
"message": format!("GetUserStatus 请求执行失败: {detail}"),
"status_code": 502,
}));
continue;
}
};
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0);
let mut metadata_update = None::<serde_json::Value>;
let (mut oauth_invalid_at_unix_secs, mut oauth_invalid_reason) =
quota_refresh_success_invalid_state(&key);
let mut status = "error".to_string();
let mut message = None::<String>;
if result.status_code == 200 {
if let Some(body_json) = result
.body
.as_ref()
.and_then(|body| body.json_body.as_ref())
{
let mut windsurf_metadata =
aether_admin::provider::quota::parse_windsurf_user_status_response(
body_json,
now_unix_secs,
);
if let Some(mut metadata) = windsurf_metadata.take() {
let model_metadata = match execute_windsurf_model_configs_plan(
state,
&transport,
api_key,
proxy_override.as_ref(),
)
.await?
{
ProviderQuotaExecutionOutcome::Response(model_result)
if model_result.status_code == 200 =>
{
model_result
.body
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(|body_json| {
aether_admin::provider::quota::parse_windsurf_model_configs_response(
body_json,
now_unix_secs,
)
})
}
ProviderQuotaExecutionOutcome::Response(model_result) => {
let detail = extract_execution_error_message(&model_result)
.unwrap_or_else(|| format!("HTTP {}", model_result.status_code));
let detail = sanitize_windsurf_probe_detail(detail);
append_windsurf_probe_warning(
&mut metadata,
"model_configs",
format!("GetCascadeModelConfigs 返回: {detail}"),
);
None
}
ProviderQuotaExecutionOutcome::Failure(detail) => {
let detail = sanitize_windsurf_probe_detail(detail);
append_windsurf_probe_warning(
&mut metadata,
"model_configs",
format!("GetCascadeModelConfigs 执行失败: {detail}"),
);
None
}
};
let rate_limit_metadata = match execute_windsurf_rate_limit_plan(
state,
&transport,
api_key,
proxy_override.as_ref(),
)
.await?
{
ProviderQuotaExecutionOutcome::Response(rate_limit_result)
if rate_limit_result.status_code == 200 =>
{
rate_limit_result
.body
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(|body_json| {
aether_admin::provider::quota::parse_windsurf_rate_limit_response(
body_json,
now_unix_secs,
)
})
}
ProviderQuotaExecutionOutcome::Response(rate_limit_result) => {
let detail = extract_execution_error_message(&rate_limit_result)
.unwrap_or_else(|| format!("HTTP {}", rate_limit_result.status_code));
let detail = sanitize_windsurf_probe_detail(detail);
append_windsurf_probe_warning(
&mut metadata,
"rate_limit",
format!("CheckUserMessageRateLimit 返回: {detail}"),
);
None
}
ProviderQuotaExecutionOutcome::Failure(detail) => {
let detail = sanitize_windsurf_probe_detail(detail);
append_windsurf_probe_warning(
&mut metadata,
"rate_limit",
format!("CheckUserMessageRateLimit 执行失败: {detail}"),
);
None
}
};
metadata = merge_windsurf_probe_metadata(
metadata,
model_metadata,
rate_limit_metadata,
);
metadata_update = Some(build_windsurf_metadata_update(
key.upstream_metadata.as_ref(),
metadata,
));
status = "success".to_string();
} else {
status = "no_metadata".to_string();
message = Some("响应中未包含 Windsurf 限额信息".to_string());
}
} else {
status = "no_metadata".to_string();
message = Some("无法解析 GetUserStatus 响应".to_string());
}
} else {
let err_msg =
extract_execution_error_message(&result).map(sanitize_windsurf_probe_detail);
message = Some(match err_msg.as_deref() {
Some(detail) if !detail.is_empty() => {
format!(
"GetUserStatus 返回状态码 {}: {}",
result.status_code, detail
)
}
_ => format!("GetUserStatus 返回状态码 {}", result.status_code),
});
let detail = err_msg
.clone()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| format!("HTTP {}", result.status_code));
let mut metadata = serde_json::Map::new();
metadata.insert("updated_at".to_string(), json!(now_unix_secs));
metadata.insert("last_error".to_string(), json!(detail));
match result.status_code {
401 | 403 => {
oauth_invalid_at_unix_secs = Some(now_unix_secs);
oauth_invalid_reason =
Some(format!("Windsurf token 无效或已被拒绝: {}", detail));
metadata.insert("banned".to_string(), json!(result.status_code == 403));
status = if result.status_code == 401 {
"auth_invalid".to_string()
} else {
"forbidden".to_string()
};
}
429 => {
metadata.insert(
"rate_limit".to_string(),
json!({
"limited": true,
"message": metadata
.get("last_error")
.cloned()
.unwrap_or_else(|| json!("rate limited")),
}),
);
status = "rate_limited".to_string();
}
_ => {}
}
metadata_update = Some(build_windsurf_metadata_update(
key.upstream_metadata.as_ref(),
serde_json::Value::Object(metadata),
));
}
if !persist_provider_quota_refresh_state(
state,
&key.id,
metadata_update.as_ref(),
oauth_invalid_at_unix_secs,
oauth_invalid_reason,
None,
)
.await?
{
failed_count += 1;
results.push(json!({
"key_id": key.id,
"key_name": key.name,
"status": "error",
"message": "Key 状态写入失败",
}));
continue;
}
if status == "success" {
success_count += 1;
} else {
failed_count += 1;
}
let mut payload = serde_json::Map::new();
payload.insert("key_id".to_string(), json!(key.id));
payload.insert("key_name".to_string(), json!(key.name));
payload.insert("status".to_string(), json!(status));
if let Some(message) = message {
payload.insert("message".to_string(), json!(message));
}
if result.status_code != 200 {
payload.insert("status_code".to_string(), json!(result.status_code));
}
if let Some(metadata) = metadata_update
.as_ref()
.and_then(|value| value.get("windsurf"))
.cloned()
{
payload.insert("metadata".to_string(), metadata);
}
if let Some(quota_snapshot) = build_quota_snapshot_payload(
"windsurf",
key.status_snapshot.as_ref(),
metadata_update.as_ref(),
) {
payload.insert("quota_snapshot".to_string(), quota_snapshot);
}
results.push(serde_json::Value::Object(payload));
}
Ok(Some(json!({
"success": success_count,
"failed": failed_count,
"total": success_count + failed_count,
"results": results,
"message": format!("已处理 {} 个 Key", success_count + failed_count),
"auto_removed": 0,
})))
}
#[cfg(test)]
mod tests {
use serde_json::json;
#[test]
fn windsurf_probe_metadata_merges_user_status_models_and_rate_limit() {
let metadata = super::merge_windsurf_probe_metadata(
json!({
"plan_name": "Pro",
"daily_remaining_percent": 42.0,
"updated_at": 1_770_000_000u64,
}),
Some(json!({
"allowed_models_count": 2u64,
"models": [
{"model_uid": "claude-sonnet-4-5"},
{"model_uid": "gpt-5-mini"}
],
"updated_at": 1_770_000_010u64,
})),
Some(json!({
"rate_limit": {
"limited": true,
"messages_remaining": 0.0,
"retry_after_ms": 60_000u64
},
"updated_at": 1_770_000_020u64,
})),
);
assert_eq!(metadata["plan_name"], json!("Pro"));
assert_eq!(metadata["daily_remaining_percent"], json!(42.0));
assert_eq!(metadata["allowed_models_count"], json!(2u64));
assert_eq!(metadata["rate_limit"]["limited"], json!(true));
assert_eq!(metadata["updated_at"], json!(1_770_000_020u64));
}
#[test]
fn windsurf_probe_detail_redacts_sensitive_values() {
let detail = super::sanitize_windsurf_probe_detail(
r#"{"error":{"message":"bad"},"apiKey":"sk-secret","sessionToken":"devin-session-token$secret"}"#,
);
assert!(detail.contains("[REDACTED]"));
assert!(!detail.contains("sk-secret"));
assert!(!detail.contains("devin-session-token$secret"));
}
#[test]
fn windsurf_metadata_update_preserves_existing_bucket_fields() {
let update = super::build_windsurf_metadata_update(
Some(&json!({
"windsurf": {
"daily_remaining_percent": 0.0,
"allowed_models_count": 3,
"updated_at": 1u64
}
})),
json!({
"last_error": "HTTP 429",
"rate_limit": {"limited": true},
"updated_at": 2u64
}),
);
assert_eq!(
update.pointer("/windsurf/daily_remaining_percent"),
Some(&json!(0.0))
);
assert_eq!(
update.pointer("/windsurf/allowed_models_count"),
Some(&json!(3))
);
assert_eq!(update.pointer("/windsurf/updated_at"), Some(&json!(2u64)));
assert_eq!(
update.pointer("/windsurf/rate_limit/limited"),
Some(&json!(true))
);
}
}

View File

@@ -32,68 +32,6 @@ pub(super) fn admin_provider_ops_is_valid_action_type(action_type: &str) -> bool
)
}
pub(crate) fn admin_provider_ops_saved_connector_credentials(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
) -> serde_json::Map<String, serde_json::Value> {
admin_provider_ops_decrypted_credentials(
state,
admin_provider_ops_config_object(provider)
.and_then(admin_provider_ops_connector_object)
.and_then(|connector| connector.get("credentials")),
)
}
pub(crate) async fn admin_provider_ops_query_balance_response_for_credentials(
state: &AdminAppState<'_>,
provider_id: &str,
provider: &StoredProviderCatalogProvider,
architecture_id: &str,
base_url: &str,
provider_ops_config: &serde_json::Map<String, serde_json::Value>,
connector_config: &serde_json::Map<String, serde_json::Value>,
credentials: &serde_json::Map<String, serde_json::Value>,
request_config: Option<&serde_json::Map<String, serde_json::Value>>,
) -> serde_json::Value {
let architecture_id = normalize_architecture_id(architecture_id);
let Some(architecture) = get_architecture(architecture_id) else {
return responses::admin_provider_ops_action_not_supported(
"query_balance",
ADMIN_PROVIDER_OPS_ACTION_RUST_ONLY_MESSAGE,
);
};
let headers = match build_headers(architecture.architecture_id, connector_config, credentials) {
Ok(headers) => headers,
Err(message) => {
return responses::admin_provider_ops_action_not_configured("query_balance", message);
}
};
let Some(action_config) = resolve_action_config(
architecture_id,
provider_ops_config,
"query_balance",
request_config,
) else {
return responses::admin_provider_ops_action_not_supported(
"query_balance",
ADMIN_PROVIDER_OPS_ACTION_RUST_ONLY_MESSAGE,
);
};
query_balance::admin_provider_ops_run_query_balance_action(
state,
provider_id,
provider,
&architecture,
base_url,
&action_config,
&headers,
credentials,
None,
)
.await
}
pub(crate) async fn admin_provider_ops_local_action_response(
state: &AdminAppState<'_>,
provider_id: &str,

View File

@@ -111,14 +111,11 @@ pub(super) async fn admin_provider_ops_run_query_balance_action(
if status != http::StatusCode::OK {
let cookie_auth = architecture.query_balance_cookie_auth_errors;
let new_api_token_auth = architecture.architecture_id == "new_api";
return match status {
http::StatusCode::UNAUTHORIZED => admin_provider_ops_action_error(
"auth_failed",
"query_balance",
if new_api_token_auth {
"访问令牌无效,请使用 New API 个人安全设置里的访问令牌"
} else if cookie_auth {
if cookie_auth {
"Cookie 已失效,请重新配置"
} else {
"认证失败"
@@ -128,9 +125,7 @@ pub(super) async fn admin_provider_ops_run_query_balance_action(
http::StatusCode::FORBIDDEN => admin_provider_ops_action_error(
"auth_failed",
"query_balance",
if new_api_token_auth {
"访问令牌无效或无权限,请使用 New API 个人安全设置里的访问令牌"
} else if cookie_auth {
if cookie_auth {
"Cookie 已失效或无权限"
} else {
"无权限访问"

View File

@@ -8,9 +8,7 @@ use super::super::responses::{
};
use super::super::support::admin_provider_ops_json_object_map;
use crate::handlers::admin::request::AdminAppState;
use aether_admin::provider::ops::{
parse_sub2api_api_key_usage_payload, parse_sub2api_balance_payload,
};
use aether_admin::provider::ops::parse_sub2api_balance_payload;
use aether_contracts::ProxySnapshot;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider;
use serde_json::{json, Value};
@@ -26,23 +24,6 @@ pub(super) async fn admin_provider_ops_sub2api_balance_payload(
proxy_snapshot: Option<&ProxySnapshot>,
) -> serde_json::Value {
let start = std::time::Instant::now();
if let Some(api_key) = credentials
.get("api_key")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
return admin_provider_ops_sub2api_api_key_balance_payload(
state,
provider_id,
base_url,
action_config,
api_key,
proxy_snapshot,
start,
)
.await;
}
let (access_token, updated_credentials, _frontend_updated_credentials) =
match admin_provider_ops_sub2api_exchange_token(
state,
@@ -210,117 +191,6 @@ pub(super) async fn admin_provider_ops_sub2api_balance_payload(
)
}
async fn admin_provider_ops_sub2api_api_key_balance_payload(
state: &AdminAppState<'_>,
provider_id: &str,
base_url: &str,
action_config: &serde_json::Map<String, serde_json::Value>,
api_key: &str,
proxy_snapshot: Option<&ProxySnapshot>,
start: std::time::Instant,
) -> serde_json::Value {
let usage_endpoint = action_config
.get("api_key_usage_endpoint")
.or_else(|| action_config.get("usage_endpoint"))
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("/v1/usage");
let usage_url = admin_provider_ops_sub2api_request_url(base_url, usage_endpoint);
let auth_value = match reqwest::header::HeaderValue::from_str(&format!("Bearer {api_key}")) {
Ok(value) => value,
Err(_) => {
return admin_provider_ops_action_error(
"parse_error",
"query_balance",
"API Key 格式无效",
Some(start.elapsed().as_millis() as u64),
);
}
};
let auth_headers = reqwest::header::HeaderMap::from_iter([
(reqwest::header::AUTHORIZATION, auth_value),
(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
),
]);
let request_id = format!("provider-ops-action:sub2api:usage:{provider_id}");
let result = admin_provider_ops_execute_json_request(
state,
&request_id,
reqwest::Method::GET,
&usage_url,
&auth_headers,
None,
proxy_snapshot,
)
.await;
let response_time_ms = Some(start.elapsed().as_millis() as u64);
let (status, response_json) = match result {
Ok(result) => result,
Err(AdminProviderOpsExecuteJsonError::InvalidJson(message))
| Err(AdminProviderOpsExecuteJsonError::Transport(message)) => {
return admin_provider_ops_action_error(
"network_error",
"query_balance",
network_error_message(&message),
response_time_ms,
);
}
};
if matches!(
status,
http::StatusCode::UNAUTHORIZED | http::StatusCode::FORBIDDEN
) {
return admin_provider_ops_action_error(
"auth_failed",
"query_balance",
"认证失败,请检查 API Key",
response_time_ms,
);
}
if status != http::StatusCode::OK {
return admin_provider_ops_action_error(
"unknown_error",
"query_balance",
format!(
"HTTP {}: {}",
status.as_u16(),
status.canonical_reason().unwrap_or("Unknown")
),
response_time_ms,
);
}
let data = match parse_sub2api_api_key_usage_payload(action_config, &response_json) {
Ok(payload) => payload,
Err(message) => {
return admin_provider_ops_action_error(
if message.contains("无效") {
"auth_failed"
} else if message == "响应格式无效" {
"parse_error"
} else {
"unknown_error"
},
"query_balance",
message,
response_time_ms,
);
}
};
admin_provider_ops_action_response(
"success",
"query_balance",
data,
None,
response_time_ms,
86400,
)
}
fn network_error_message(error: &str) -> String {
let normalized = error.trim();
let lower = normalized.to_ascii_lowercase();

View File

@@ -70,7 +70,7 @@ pub(super) async fn read_admin_provider_ops_balance_cache(
}
}
pub(super) async fn store_admin_provider_ops_balance_cache(
pub(crate) async fn store_admin_provider_ops_balance_cache(
state: &AdminAppState<'_>,
provider_id: &str,
payload: &Value,

View File

@@ -1,4 +1,7 @@
use super::support::{AdminProviderOpsSaveConfigRequest, ADMIN_PROVIDER_OPS_SENSITIVE_FIELDS};
use super::support::{
AdminProviderOpsQuotaAlertConfigRequest, AdminProviderOpsSaveConfigRequest,
ADMIN_PROVIDER_OPS_SENSITIVE_FIELDS,
};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::provider::ops as admin_provider_ops_pure;
@@ -8,6 +11,10 @@ use aether_data_contracts::repository::provider_catalog::{
use serde_json::json;
use std::time::{SystemTime, UNIX_EPOCH};
const PROVIDER_OPS_QUOTA_ALERT_DEFAULT_FETCH_INTERVAL_SECS: u64 = 30;
const PROVIDER_OPS_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS: u64 = 30;
const PROVIDER_OPS_QUOTA_ALERT_MAX_FETCH_INTERVAL_SECS: u64 = 86_400;
pub(super) fn admin_provider_ops_config_object(
provider: &StoredProviderCatalogProvider,
) -> Option<&serde_json::Map<String, serde_json::Value>> {
@@ -250,9 +257,6 @@ pub(super) fn build_admin_provider_ops_saved_config_value(
provider: &StoredProviderCatalogProvider,
payload: AdminProviderOpsSaveConfigRequest,
) -> Result<serde_json::Value, String> {
let architecture_id =
admin_provider_ops_pure::normalize_architecture_id(payload.architecture_id.as_str())
.to_string();
let auth_type = payload.connector.auth_type.trim().to_string();
if auth_type.is_empty() || !admin_provider_ops_is_supported_auth_type(auth_type.as_str()) {
return Err("connector.auth_type 必须是合法的认证类型".to_string());
@@ -260,7 +264,7 @@ pub(super) fn build_admin_provider_ops_saved_config_value(
let merged_credentials = admin_provider_ops_merge_credentials(
state,
architecture_id.as_str(),
payload.architecture_id.as_str(),
provider,
payload.connector.credentials,
);
@@ -279,9 +283,10 @@ pub(super) fn build_admin_provider_ops_saved_config_value(
)
})
.collect::<serde_json::Map<String, serde_json::Value>>();
let quota_alert = normalize_admin_provider_ops_quota_alert(payload.quota_alert)?;
Ok(json!({
"architecture_id": architecture_id,
"architecture_id": payload.architecture_id,
"base_url": payload.base_url,
"connector": {
"auth_type": auth_type,
@@ -290,9 +295,48 @@ pub(super) fn build_admin_provider_ops_saved_config_value(
},
"actions": actions,
"schedule": payload.schedule,
"quota_alert": quota_alert,
}))
}
fn normalize_admin_provider_ops_quota_alert(
request: Option<AdminProviderOpsQuotaAlertConfigRequest>,
) -> Result<serde_json::Value, String> {
let Some(request) = request else {
return Ok(default_admin_provider_ops_quota_alert());
};
let threshold_amount = request.threshold_amount.unwrap_or(0.0);
if threshold_amount < 0.0 {
return Err("quota_alert.threshold_amount 必须大于等于 0".to_string());
}
let fetch_interval_seconds = request
.fetch_interval_seconds
.unwrap_or(PROVIDER_OPS_QUOTA_ALERT_DEFAULT_FETCH_INTERVAL_SECS);
if !(PROVIDER_OPS_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS
..=PROVIDER_OPS_QUOTA_ALERT_MAX_FETCH_INTERVAL_SECS)
.contains(&fetch_interval_seconds)
{
return Err(format!(
"quota_alert.fetch_interval_seconds 必须在 {}{} 秒之间",
PROVIDER_OPS_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS,
PROVIDER_OPS_QUOTA_ALERT_MAX_FETCH_INTERVAL_SECS
));
}
Ok(json!({
"enabled": request.enabled,
"threshold_amount": threshold_amount,
"fetch_interval_seconds": fetch_interval_seconds,
}))
}
fn default_admin_provider_ops_quota_alert() -> serde_json::Value {
json!({
"enabled": false,
"threshold_amount": 0.0,
"fetch_interval_seconds": PROVIDER_OPS_QUOTA_ALERT_DEFAULT_FETCH_INTERVAL_SECS,
})
}
pub(super) fn resolve_admin_provider_ops_base_url(
provider: &StoredProviderCatalogProvider,
endpoints: &[StoredProviderCatalogEndpoint],
@@ -331,16 +375,14 @@ pub(super) fn build_admin_provider_ops_config_payload(
});
};
let connector = admin_provider_ops_connector_object(provider_ops_config);
let architecture_id = provider_ops_config
.get("architecture_id")
.and_then(serde_json::Value::as_str)
.map(admin_provider_ops_pure::normalize_architecture_id)
.unwrap_or("generic_api");
json!({
"provider_id": provider_id,
"is_configured": true,
"architecture_id": architecture_id,
"architecture_id": provider_ops_config
.get("architecture_id")
.and_then(serde_json::Value::as_str)
.unwrap_or("generic_api"),
"base_url": resolve_admin_provider_ops_base_url(
provider,
endpoints,
@@ -361,5 +403,10 @@ pub(super) fn build_admin_provider_ops_config_payload(
connector.and_then(|connector| connector.get("credentials")),
),
},
"quota_alert": provider_ops_config
.get("quota_alert")
.filter(|value| value.is_object())
.cloned()
.unwrap_or_else(default_admin_provider_ops_quota_alert),
})
}

View File

@@ -4,4 +4,5 @@ mod config;
mod routes;
mod support;
mod verify;
pub(crate) use self::balance_cache::store_admin_provider_ops_balance_cache;
pub(super) use self::routes::maybe_build_local_admin_provider_ops_providers_response;

View File

@@ -33,6 +33,8 @@ pub(super) struct AdminProviderOpsSaveConfigRequest {
pub(crate) actions: BTreeMap<String, AdminProviderOpsActionConfigRequest>,
#[serde(default)]
pub(crate) schedule: BTreeMap<String, String>,
#[serde(default)]
pub(crate) quota_alert: Option<AdminProviderOpsQuotaAlertConfigRequest>,
}
#[derive(Debug, Deserialize)]
@@ -52,6 +54,16 @@ pub(super) struct AdminProviderOpsActionConfigRequest {
pub(crate) config: serde_json::Map<String, serde_json::Value>,
}
#[derive(Debug, Deserialize)]
pub(super) struct AdminProviderOpsQuotaAlertConfigRequest {
#[serde(default)]
pub(crate) enabled: bool,
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub(crate) threshold_amount: Option<f64>,
#[serde(default)]
pub(crate) fetch_interval_seconds: Option<u64>,
}
#[derive(Debug, Deserialize)]
pub(super) struct AdminProviderOpsConnectRequest {
#[serde(default)]
@@ -71,3 +83,28 @@ fn default_admin_provider_ops_architecture_id() -> String {
fn default_admin_provider_ops_action_enabled() -> bool {
true
}
fn deserialize_optional_f64_from_number<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = Option::<serde_json::Value>::deserialize(deserializer)?;
match value {
None | Some(serde_json::Value::Null) => Ok(None),
Some(serde_json::Value::Number(number)) => number
.as_f64()
.filter(|value| value.is_finite())
.map(Some)
.ok_or_else(|| serde::de::Error::custom("expected a finite number")),
Some(serde_json::Value::String(raw)) => raw
.trim()
.parse::<f64>()
.ok()
.filter(|value| value.is_finite())
.map(Some)
.ok_or_else(|| serde::de::Error::custom("expected a finite number or numeric string")),
Some(_) => Err(serde::de::Error::custom(
"expected a finite number or numeric string",
)),
}
}

View File

@@ -6,8 +6,7 @@ use crate::handlers::admin::provider::ops::providers::config::persist_admin_prov
use crate::handlers::admin::request::AdminAppState;
use aether_admin::provider::ops::{
admin_provider_ops_frontend_updated_credentials, admin_provider_ops_verify_failure,
admin_provider_ops_verify_success, admin_provider_ops_verify_user_payload,
parse_sub2api_api_key_usage_payload, parse_verify_payload, ADMIN_PROVIDER_OPS_USER_AGENT,
parse_verify_payload, ADMIN_PROVIDER_OPS_USER_AGENT,
};
use aether_contracts::ProxySnapshot;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider;
@@ -22,21 +21,6 @@ pub(super) async fn admin_provider_ops_local_sub2api_verify_response(
credentials: &Map<String, Value>,
proxy_snapshot: Option<&ProxySnapshot>,
) -> Value {
if let Some(api_key) = credentials
.get("api_key")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
return admin_provider_ops_local_sub2api_api_key_verify_response(
state,
base_url,
api_key,
proxy_snapshot,
)
.await;
}
let (access_token, updated_credentials, frontend_updated_credentials) =
match admin_provider_ops_sub2api_exchange_token(
state,
@@ -109,78 +93,6 @@ pub(super) async fn admin_provider_ops_local_sub2api_verify_response(
)
}
async fn admin_provider_ops_local_sub2api_api_key_verify_response(
state: &AdminAppState<'_>,
base_url: &str,
api_key: &str,
proxy_snapshot: Option<&ProxySnapshot>,
) -> Value {
let usage_url = admin_provider_ops_sub2api_request_url(base_url, "/v1/usage");
let auth_value = match reqwest::header::HeaderValue::from_str(&format!("Bearer {api_key}")) {
Ok(value) => value,
Err(_) => return admin_provider_ops_verify_failure("API Key 格式无效"),
};
let auth_headers = reqwest::header::HeaderMap::from_iter([
(reqwest::header::AUTHORIZATION, auth_value),
(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
),
]);
let auth_headers =
admin_provider_ops_headers_with_transport_controls(&auth_headers, None, true);
let (status, response_json) = match admin_provider_ops_execute_json_request(
state,
"provider-ops-verify:sub2api:api-key",
reqwest::Method::GET,
&usage_url,
&auth_headers,
None,
proxy_snapshot,
)
.await
{
Ok(result) => result,
Err(AdminProviderOpsExecuteJsonError::InvalidJson(message))
| Err(AdminProviderOpsExecuteJsonError::Transport(message)) => {
return admin_provider_ops_verify_failure(
admin_provider_ops_verify_execution_error_message(&message),
);
}
};
if matches!(
status,
http::StatusCode::UNAUTHORIZED | http::StatusCode::FORBIDDEN
) {
return admin_provider_ops_verify_failure("认证失败API Key 无效或已过期");
}
if status != http::StatusCode::OK {
return admin_provider_ops_verify_failure(format!("验证失败HTTP {}", status.as_u16()));
}
let payload = match parse_sub2api_api_key_usage_payload(&Map::new(), &response_json) {
Ok(payload) => payload,
Err(message) => return admin_provider_ops_verify_failure(message),
};
let quota = payload.get("total_available").and_then(Value::as_f64);
let extra = payload
.get("extra")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
admin_provider_ops_verify_success(
admin_provider_ops_verify_user_payload(
Some("Sub2API API Key".to_string()),
Some("Sub2API API Key".to_string()),
None,
quota,
Some(extra),
),
None,
)
}
// 对齐 Python httpx.AsyncClient(base_url=...) 的行为:
// 以 "/" 开头的端点始终相对站点根路径解析,而不是简单字符串拼接。
pub(in super::super) fn admin_provider_ops_sub2api_request_url(

View File

@@ -14,7 +14,7 @@ use super::{provider_query_key_display_name, provider_query_provider_payload};
use crate::ai_serving::{
maybe_build_sync_finalize_outcome, GatewayControlDecision,
ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME, GEMINI_CHAT_SYNC_FINALIZE_REPORT_KIND,
OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
OPENAI_CHAT_SYNC_FINALIZE_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
};
use crate::clock::current_unix_ms;
use crate::execution_runtime;
@@ -1486,6 +1486,17 @@ fn provider_query_decode_execution_body(
.and_then(|value| base64::engine::general_purpose::STANDARD.decode(value).ok())
}
fn provider_query_execution_json_body(result: &aether_contracts::ExecutionResult) -> Option<Value> {
result
.body
.as_ref()
.and_then(|body| body.json_body.clone())
.or_else(|| {
provider_query_decode_execution_body(result)
.and_then(|body| serde_json::from_slice::<Value>(&body).ok())
})
}
fn provider_query_aggregate_standard_stream_sync_response(
provider_api_format: &str,
body: &[u8],
@@ -1505,15 +1516,11 @@ fn provider_query_standard_execution_response_body(
provider_api_format: &str,
result: &aether_contracts::ExecutionResult,
) -> Option<Value> {
let body = result
.body
.as_ref()
.and_then(|body| body.json_body.clone())
.or_else(|| {
provider_query_decode_execution_body(result).and_then(|body| {
provider_query_aggregate_standard_stream_sync_response(provider_api_format, &body)
})
})?;
let body = provider_query_execution_json_body(result).or_else(|| {
provider_query_decode_execution_body(result).and_then(|body| {
provider_query_aggregate_standard_stream_sync_response(provider_api_format, &body)
})
})?;
if result.status_code < 400
&& provider_query_normalize_api_format_alias(provider_api_format)
== "gemini:generate_content"
@@ -1527,10 +1534,8 @@ fn provider_query_standard_execution_response_body(
fn provider_query_extract_error_message(
result: &aether_contracts::ExecutionResult,
) -> Option<String> {
result
.body
provider_query_execution_json_body(result)
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(Value::as_object)
.and_then(|value| {
value
@@ -1589,7 +1594,7 @@ async fn provider_query_finalize_kiro_result(
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
body_json: provider_query_execution_json_body(result),
client_body_json: None,
body_base64: result
.body
@@ -1781,6 +1786,56 @@ async fn provider_query_execute_kiro_test_candidate(
})
}
async fn provider_query_finalize_windsurf_result(
route_path: &str,
trace_id: &str,
requested_model: &str,
mapped_model: &str,
original_request_body: &Value,
result: &aether_contracts::ExecutionResult,
) -> Result<Option<Value>, GatewayError> {
let decision = GatewayControlDecision::synthetic(
route_path,
Some("admin_proxy".to_string()),
Some("provider_query_manage".to_string()),
Some("test_model_failover".to_string()),
Some("openai:chat".to_string()),
);
let payload = GatewaySyncReportRequest {
trace_id: trace_id.to_string(),
report_kind: OPENAI_CHAT_SYNC_FINALIZE_REPORT_KIND.to_string(),
report_context: Some(json!({
"client_api_format": "openai:chat",
"provider_api_format": "openai:chat",
"model": requested_model,
"mapped_model": mapped_model,
"needs_conversion": false,
"has_envelope": true,
"envelope_name": crate::provider_transport::windsurf::WINDSURF_ENVELOPE_NAME,
"original_request_body": original_request_body,
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
client_body_json: None,
body_base64: result
.body
.as_ref()
.and_then(|body| body.body_bytes_b64.clone()),
telemetry: result.telemetry.clone(),
};
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
return Ok(None);
};
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
serde_json::from_slice::<Value>(&bytes)
.map(Some)
.map_err(|err| GatewayError::Internal(err.to_string()))
}
fn provider_query_build_openai_image_test_request_body_for_route(
payload: &Value,
model: &str,
@@ -1821,6 +1876,7 @@ fn provider_query_chatgpt_web_image_internal_url(base_url: &str) -> String {
fn provider_query_openai_image_test_upstream_url(
transport: &AdminGatewayProviderTransportSnapshot,
request_path: Option<&str>,
request_query: Option<&str>,
) -> String {
if transport
@@ -1841,7 +1897,11 @@ fn provider_query_openai_image_test_upstream_url(
crate::provider_transport::GROK_CHAT_PATH,
)
} else {
crate::provider_transport::build_openai_image_upstream_url(transport, request_query)
crate::provider_transport::build_openai_image_upstream_url(
transport,
request_path,
request_query,
)
}
}
@@ -1874,7 +1934,7 @@ async fn provider_query_finalize_openai_image_result(
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
body_json: provider_query_execution_json_body(result),
client_body_json: None,
body_base64: result
.body
@@ -1967,13 +2027,23 @@ async fn provider_query_execute_openai_image_test_candidate(
.provider_type
.trim()
.eq_ignore_ascii_case("grok");
let is_codex = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("codex");
let mut provider_request_body = if is_chatgpt_web {
match crate::ai_serving::build_chatgpt_web_image_request_body(&parts, &request_body, None) {
Ok(body) => body,
Err(err) => err.to_error_json(),
}
} else {
} else if is_codex || is_grok {
crate::ai_serving::build_openai_image_provider_request_body(&normalized_request)
} else {
crate::ai_serving::build_openai_image_api_provider_request_body(
&normalized_request,
Some(candidate.effective_model.as_str()),
)
};
if !is_chatgpt_web {
crate::ai_serving::apply_codex_openai_responses_special_body_edits(
@@ -2075,7 +2145,11 @@ async fn provider_query_execute_openai_image_test_candidate(
} else {
normalized_request.summary_json.clone()
};
let request_url = provider_query_openai_image_test_upstream_url(&transport, parts.uri.query());
let request_url = provider_query_openai_image_test_upstream_url(
&transport,
Some(parts.uri.path()),
parts.uri.query(),
);
let upstream_is_stream = provider_request_body
.get("stream")
.and_then(Value::as_bool)
@@ -2158,9 +2232,9 @@ async fn provider_query_execute_openai_image_test_candidate(
&result,
)
.await?
.or_else(|| result.body.as_ref().and_then(|body| body.json_body.clone()))
.or_else(|| provider_query_execution_json_body(&result))
} else {
result.body.as_ref().and_then(|body| body.json_body.clone())
provider_query_execution_json_body(&result)
};
let did_fail = result.status_code >= 400;
let error_message = if did_fail {
@@ -2635,6 +2709,22 @@ async fn provider_query_execute_standard_test_candidate(
route_path,
client_api_format,
);
if crate::provider_transport::is_windsurf_provider_transport(&transport)
&& provider_query_normalize_api_format_alias(candidate.endpoint.api_format.as_str())
== "openai:chat"
{
return provider_query_execute_windsurf_test_candidate(
state,
provider,
candidate,
payload,
route_path,
trace_id,
transport,
original_request_body,
)
.await;
}
if !provider_query_transport_supports_model_test_execution(
state,
&transport,
@@ -3042,6 +3132,182 @@ async fn provider_query_execute_standard_test_candidate(
})
}
#[allow(clippy::too_many_arguments)]
async fn provider_query_execute_windsurf_test_candidate(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
candidate: &ProviderQueryTestCandidate,
payload: &Value,
route_path: &str,
trace_id: &str,
transport: AdminGatewayProviderTransportSnapshot,
original_request_body: Value,
) -> Result<ProviderQueryExecutionOutcome, GatewayError> {
if let Some(_reason) =
crate::provider_transport::local_windsurf_request_transport_unsupported_reason_with_network(
&transport,
)
{
return Ok(provider_query_skipped_execution_outcome(
original_request_body,
provider_query_standard_test_unsupported_reason(
&transport,
candidate.endpoint.api_format.as_str(),
),
));
}
let incoming_request_headers = provider_query_extract_request_headers(payload);
let request_body = original_request_body.clone();
let request_model =
provider_query_request_body_model(&request_body, &candidate.effective_model);
let client_is_stream = request_body
.get("stream")
.and_then(Value::as_bool)
.unwrap_or(false);
let hard_requires_streaming = crate::ai_serving::force_upstream_streaming_for_provider(
transport.provider.provider_type.as_str(),
candidate.endpoint.api_format.as_str(),
);
let upstream_is_stream = crate::ai_serving::resolve_upstream_is_stream_from_endpoint_config(
transport.endpoint.config.as_ref(),
client_is_stream,
hard_requires_streaming,
);
let Some((auth_header, auth_value)) =
crate::provider_transport::windsurf::resolve_windsurf_cascade_auth(&transport).or_else(
|| crate::provider_transport::auth::resolve_local_openai_bearer_auth(&transport),
)
else {
return Ok(provider_query_skipped_execution_outcome(
request_body,
"Provider auth is unavailable for windsurf".to_string(),
));
};
let mut synthetic_request = http::Request::builder()
.uri(route_path)
.body(())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
*synthetic_request.headers_mut() = incoming_request_headers;
let (parts, _) = synthetic_request.into_parts();
let Some(provider_request_body) =
crate::provider_transport::build_windsurf_cascade_request_body(
&request_body,
request_model,
&auth_value,
transport.endpoint.body_rules.as_ref(),
Some(&parts.headers),
upstream_is_stream,
)
else {
return Ok(provider_query_skipped_execution_outcome(
request_body,
"Provider request body could not be built for windsurf".to_string(),
));
};
let Some(request_url) = crate::provider_transport::build_windsurf_cascade_upstream_url(
transport.endpoint.base_url.as_str(),
parts.uri.query(),
) else {
return Ok(provider_query_skipped_execution_outcome(
provider_request_body,
"Provider request URL is unavailable for windsurf".to_string(),
));
};
let Some(request_headers) = crate::provider_transport::build_windsurf_cascade_headers(
&parts.headers,
&provider_request_body,
&request_body,
transport.endpoint.header_rules.as_ref(),
&auth_header,
&auth_value,
upstream_is_stream,
) else {
return Ok(ProviderQueryExecutionOutcome {
status: "failed",
skip_reason: None,
error_message: Some("provider request headers build failed".to_string()),
status_code: None,
latency_ms: None,
request_url,
request_headers: BTreeMap::new(),
request_body: provider_request_body,
response_headers: BTreeMap::new(),
response_body: None,
});
};
let plan = ExecutionPlan {
request_id: trace_id.to_string(),
candidate_id: Some(format!("provider-query-{}", candidate.key.id)),
provider_name: Some(provider.name.clone()),
provider_id: provider.id.clone(),
endpoint_id: candidate.endpoint.id.clone(),
key_id: candidate.key.id.clone(),
method: "POST".to_string(),
url: request_url.clone(),
headers: request_headers.clone(),
content_type: Some("application/connect+json".to_string()),
content_encoding: None,
body: RequestBody::from_json(provider_request_body.clone()),
stream: upstream_is_stream,
client_api_format: "openai:chat".to_string(),
provider_api_format: candidate.endpoint.api_format.clone(),
model_name: Some(request_model.to_string()),
proxy: state
.resolve_transport_proxy_snapshot_with_tunnel_affinity(&transport)
.await,
transport_profile: state.resolve_transport_profile(&transport),
timeouts: state.resolve_transport_execution_timeouts(&transport),
};
let result = state
.execute_execution_runtime_sync_plan(Some(trace_id), &plan)
.await?;
let response_body = if result.status_code < 400 {
provider_query_finalize_windsurf_result(
route_path,
trace_id,
request_model,
request_model,
&request_body,
&result,
)
.await?
} else {
result.body.as_ref().and_then(|body| body.json_body.clone())
};
let missing_success_body = result.status_code < 400 && response_body.is_none();
let did_fail = result.status_code >= 400 || missing_success_body;
let error_message = if did_fail {
provider_query_extract_error_message(&result).or_else(|| {
missing_success_body.then(|| {
format!(
"Provider returned HTTP {} without a model-test response body",
result.status_code
)
})
})
} else {
None
};
Ok(ProviderQueryExecutionOutcome {
status: if did_fail { "failed" } else { "success" },
skip_reason: None,
error_message,
status_code: Some(result.status_code),
latency_ms: result.telemetry.as_ref().and_then(|value| value.elapsed_ms),
request_url,
request_headers,
request_body: provider_request_body,
response_headers: result.headers,
response_body,
})
}
async fn build_admin_provider_query_kiro_failover_response(
state: &AdminAppState<'_>,
payload: &Value,

View File

@@ -46,6 +46,22 @@ pub(super) fn provider_query_standard_test_unsupported_reason(
api_format: &str,
) -> String {
let normalized_api_format = crate::ai_serving::normalize_api_format_alias(api_format);
if crate::provider_transport::is_windsurf_provider_transport(transport)
&& normalized_api_format == "openai:chat"
{
let reason =
crate::provider_transport::local_windsurf_request_transport_unsupported_reason_with_network(
transport,
);
return match reason {
Some(reason) => format!(
"{} ({reason})",
provider_query_unsupported_test_api_format_message(api_format)
),
None => provider_query_unsupported_test_api_format_message(api_format),
};
}
let reason = match normalized_api_format.as_str() {
"openai:chat" => {
crate::provider_transport::policy::local_openai_chat_transport_unsupported_reason(
@@ -294,6 +310,15 @@ pub(super) fn provider_query_transport_supports_model_test_execution(
transport: &AdminGatewayProviderTransportSnapshot,
api_format: &str,
) -> bool {
if crate::provider_transport::is_windsurf_provider_transport(transport)
&& provider_query_normalize_api_format_alias(api_format) == "openai:chat"
{
return crate::provider_transport::local_windsurf_request_transport_unsupported_reason_with_network(
transport,
)
.is_none();
}
match provider_query_test_adapter_for_provider_api_format(
transport.provider.provider_type.as_str(),
api_format,

View File

@@ -42,9 +42,9 @@ pub(super) fn provider_query_test_attempt_payload(
"status_code": execution.status_code,
"latency_ms": execution.latency_ms,
"request_url": execution.request_url,
"request_headers": provider_query_redact_diagnostic_headers(&execution.request_headers),
"request_body": execution.request_body,
"response_headers": provider_query_redact_diagnostic_headers(&execution.response_headers),
"request_headers": redacted_provider_query_headers(&execution.request_headers),
"request_body": redacted_provider_query_value(&execution.request_body),
"response_headers": redacted_provider_query_headers(&execution.response_headers),
"response_body": execution.response_body,
})
}
@@ -172,34 +172,84 @@ fn provider_query_endpoint_route_payload(
})
}
fn provider_query_redact_diagnostic_headers(
headers: &BTreeMap<String, String>,
) -> BTreeMap<String, String> {
fn redacted_provider_query_headers(headers: &BTreeMap<String, String>) -> BTreeMap<String, String> {
headers
.iter()
.map(|(name, value)| {
if provider_query_header_is_sensitive(name) {
(name.clone(), "<redacted>".to_string())
.map(|(key, value)| {
if provider_query_field_is_sensitive(key) {
(key.clone(), "[REDACTED]".to_string())
} else {
(name.clone(), value.clone())
(key.clone(), value.clone())
}
})
.collect()
}
fn provider_query_header_is_sensitive(name: &str) -> bool {
fn redacted_provider_query_value(value: &Value) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.map(|(key, value)| {
if provider_query_field_is_sensitive(key) {
(key.clone(), Value::String("[REDACTED]".to_string()))
} else {
(key.clone(), redacted_provider_query_value(value))
}
})
.collect(),
),
Value::Array(items) => Value::Array(
items
.iter()
.map(redacted_provider_query_value)
.collect::<Vec<_>>(),
),
other => other.clone(),
}
}
fn provider_query_field_is_sensitive(key: &str) -> bool {
let key = key.trim().to_ascii_lowercase();
let normalized = key
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.collect::<String>();
if matches!(
normalized.as_str(),
"maxtokens"
| "maxoutputtokens"
| "inputtokens"
| "outputtokens"
| "prompttokens"
| "completiontokens"
| "totaltokens"
) {
return false;
}
matches!(
name.trim().to_ascii_lowercase().as_str(),
key.as_str(),
"authorization"
| "proxy-authorization"
| "cookie"
| "set-cookie"
| "x-api-key"
| "api_key"
| "apikey"
| "api-key"
| "x-api-key"
| "x-goog-api-key"
| "anthropic-api-key"
| "openai-api-key"
)
| "x-codeium-csrf-token"
| "access_token"
| "refresh_token"
| "id_token"
| "password"
| "secret"
) || normalized.ends_with("token")
|| normalized.contains("secret")
|| normalized.contains("apikey")
|| normalized.contains("authorization")
}
pub(super) fn provider_query_candidate_summary_payload(
@@ -309,34 +359,93 @@ pub(super) fn provider_query_candidate_summary_payload(
#[cfg(test)]
mod tests {
use super::*;
use super::{redacted_provider_query_headers, redacted_provider_query_value};
use serde_json::json;
use std::collections::BTreeMap;
#[test]
fn provider_query_diagnostic_headers_redact_credentials() {
fn redacts_sensitive_provider_query_headers() {
let headers = BTreeMap::from([
("cookie".to_string(), "sso=secret".to_string()),
("authorization".to_string(), "Bearer secret".to_string()),
(
"authorization".to_string(),
"Bearer secret-token".to_string(),
),
("x-goog-api-key".to_string(), "secret".to_string()),
("content-type".to_string(), "application/json".to_string()),
(
"x-codeium-csrf-token".to_string(),
"csrf-secret".to_string(),
),
]);
let redacted = provider_query_redact_diagnostic_headers(&headers);
let redacted = redacted_provider_query_headers(&headers);
assert_eq!(
redacted.get("cookie").map(String::as_str),
Some("<redacted>")
Some("[REDACTED]")
);
assert_eq!(
redacted.get("authorization").map(String::as_str),
Some("<redacted>")
Some("[REDACTED]")
);
assert_eq!(
redacted.get("x-goog-api-key").map(String::as_str),
Some("<redacted>")
Some("[REDACTED]")
);
assert_eq!(
redacted.get("x-codeium-csrf-token").map(String::as_str),
Some("[REDACTED]")
);
assert_eq!(
redacted.get("content-type").map(String::as_str),
Some("application/json")
);
}
#[test]
fn redacts_sensitive_provider_query_request_body_fields() {
let body = json!({
"metadata": {
"apiKey": "devin-session-token$secret",
"ideName": "windsurf"
},
"messages": [{"role": "user", "content": "hello"}],
"stream": true
});
let redacted = redacted_provider_query_value(&body);
assert_eq!(
redacted.pointer("/metadata/apiKey"),
Some(&json!("[REDACTED]"))
);
assert_eq!(
redacted.pointer("/metadata/ideName"),
Some(&json!("windsurf"))
);
assert_eq!(redacted.pointer("/stream"), Some(&json!(true)));
}
#[test]
fn keeps_non_secret_token_count_fields_visible() {
let body = json!({
"maxTokens": 64,
"usage": {
"inputTokens": 10,
"outputTokens": 2,
"accessToken": "secret"
}
});
let redacted = redacted_provider_query_value(&body);
assert_eq!(redacted.pointer("/maxTokens"), Some(&json!(64)));
assert_eq!(redacted.pointer("/usage/inputTokens"), Some(&json!(10)));
assert_eq!(redacted.pointer("/usage/outputTokens"), Some(&json!(2)));
assert_eq!(
redacted.pointer("/usage/accessToken"),
Some(&json!("[REDACTED]"))
);
}
}

View File

@@ -90,6 +90,44 @@ fn provider_query_test_request_body_defaults_missing_model() {
assert_eq!(body["model"], json!("fallback-model"));
}
#[test]
fn provider_query_execution_json_body_decodes_stream_encoded_json_response() {
use base64::Engine as _;
let body = json!({
"created": 1,
"data": [{
"url": "https://example.test/image.png"
}]
});
let encoded_body = base64::engine::general_purpose::STANDARD
.encode(serde_json::to_vec(&body).expect("test body should serialize"));
let result = aether_contracts::ExecutionResult {
request_id: "request-1".to_string(),
candidate_id: None,
status_code: 200,
headers: std::collections::BTreeMap::from([(
"content-type".to_string(),
"application/json".to_string(),
)]),
body: Some(aether_contracts::ResponseBody {
json_body: None,
body_bytes_b64: Some(encoded_body),
}),
telemetry: None,
error: None,
};
assert_eq!(
provider_query_execution_json_body(&result),
Some(body.clone())
);
assert_eq!(
provider_query_standard_execution_response_body("openai:image", &result),
Some(body)
);
}
#[test]
fn provider_query_test_request_body_fills_empty_conversation() {
let payload = json!({
@@ -850,17 +888,39 @@ fn provider_query_grok_image_test_uses_grok_app_chat_upstream_url() {
let transport = sample_openai_image_transport("grok");
assert_eq!(
provider_query_openai_image_test_upstream_url(&transport, Some("trace=1")),
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/rest/app-chat/conversations/new"
);
}
#[test]
fn provider_query_custom_image_test_uses_images_upstream_url() {
let transport = sample_openai_image_transport("custom");
assert_eq!(
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/v1/images/generations?trace=1"
);
}
#[test]
fn provider_query_chatgpt_web_image_test_uses_internal_upstream_url() {
let transport = sample_openai_image_transport("chatgpt_web");
assert_eq!(
provider_query_openai_image_test_upstream_url(&transport, Some("trace=1")),
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/__aether/chatgpt-web-image"
);
}

View File

@@ -12,13 +12,6 @@ pub(crate) fn admin_provider_id_for_refresh_quota(request_path: &str) -> Option<
.map(ToOwned::to_owned)
}
pub(crate) fn admin_provider_id_for_key_balance(request_path: &str) -> Option<String> {
request_path
.strip_prefix("/api/admin/endpoints/providers/")?
.strip_suffix("/key-balance")
.map(ToOwned::to_owned)
}
pub(crate) fn admin_reveal_key_id(request_path: &str) -> Option<String> {
request_path
.strip_prefix("/api/admin/endpoints/keys/")?

View File

@@ -15,9 +15,9 @@ pub(crate) use self::crud::{
is_admin_providers_root,
};
pub(crate) use self::endpoint_keys::{
admin_clear_oauth_invalid_key_id, admin_export_key_id, admin_provider_id_for_key_balance,
admin_provider_id_for_keys, admin_provider_id_for_refresh_quota,
admin_reset_cycle_stats_key_id, admin_reveal_key_id, admin_update_key_id,
admin_clear_oauth_invalid_key_id, admin_export_key_id, admin_provider_id_for_keys,
admin_provider_id_for_refresh_quota, admin_reset_cycle_stats_key_id, admin_reveal_key_id,
admin_update_key_id,
};
pub(crate) use self::oauth::{
admin_provider_oauth_batch_import_provider_id, admin_provider_oauth_batch_import_task_path,

View File

@@ -31,7 +31,7 @@ pub(crate) async fn build_admin_provider_summary_payload(
active_global_model_ids_result,
) = tokio::join!(
state.list_provider_catalog_endpoints_by_provider_ids(&provider_ids),
state.list_provider_catalog_keys_by_provider_ids(&provider_ids),
state.list_provider_catalog_key_summaries_by_provider_ids(&provider_ids),
state.read_provider_quota_snapshot(provider_id),
state.list_provider_model_stats(&provider_ids),
state.list_active_global_model_ids_by_provider_ids(&provider_ids),
@@ -197,7 +197,7 @@ pub(crate) async fn build_admin_providers_summary_payload(
} else {
let (endpoints_result, keys_result, model_stats_result, active_global_model_refs_result) = tokio::join!(
state.list_provider_catalog_endpoints_by_provider_ids(&provider_ids),
state.list_provider_catalog_keys_by_provider_ids(&provider_ids),
state.list_provider_catalog_key_summaries_by_provider_ids(&provider_ids),
state.list_provider_model_stats(&provider_ids),
state.list_active_global_model_ids_by_provider_ids(&provider_ids),
);

View File

@@ -7,7 +7,7 @@ use aether_data_contracts::repository::candidates::{
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use serde_json::{json, Map, Value};
use serde_json::json;
use std::collections::BTreeMap;
fn json_truthy(value: &serde_json::Value) -> bool {
@@ -27,80 +27,6 @@ fn endpoint_timestamp_or_now(value: Option<u64>, now_unix_secs: u64) -> serde_js
.unwrap_or(serde_json::Value::Null)
}
fn finite_json_number(value: Option<&Value>) -> Option<f64> {
match value {
Some(Value::Number(number)) => number.as_f64().filter(|value| value.is_finite()),
Some(Value::String(value)) => value
.trim()
.parse::<f64>()
.ok()
.filter(|value| value.is_finite()),
_ => None,
}
}
fn finite_json_u64(value: Option<&Value>) -> Option<u64> {
finite_json_number(value).and_then(|value| {
if value >= 0.0 {
Some(value as u64)
} else {
None
}
})
}
fn latest_key_balance_summary(keys: &[StoredProviderCatalogKey]) -> Value {
let mut selected: Option<(u64, &StoredProviderCatalogKey, &Map<String, Value>)> = None;
for key in keys {
let Some(balance) = key
.upstream_metadata
.as_ref()
.and_then(|metadata| metadata.get("balance_query"))
.and_then(Value::as_object)
else {
continue;
};
let Some(updated_at) = finite_json_u64(balance.get("updated_at")) else {
continue;
};
let has_balance_value = ["total_available", "total_used", "total_granted"]
.into_iter()
.any(|field| finite_json_number(balance.get(field)).is_some());
if !has_balance_value {
continue;
}
if selected
.as_ref()
.is_none_or(|(selected_updated_at, _, _)| updated_at > *selected_updated_at)
{
selected = Some((updated_at, key, balance));
}
}
let Some((updated_at, key, balance)) = selected else {
return Value::Null;
};
json!({
"key_id": key.id.clone(),
"key_name": key.name.clone(),
"updated_at": updated_at,
"architecture_id": balance.get("architecture_id").cloned().unwrap_or(Value::Null),
"status": balance.get("status").cloned().unwrap_or_else(|| json!("success")),
"executed_at": balance.get("executed_at").cloned().unwrap_or(Value::Null),
"response_time_ms": balance.get("response_time_ms").cloned().unwrap_or(Value::Null),
"total_available": balance.get("total_available").cloned().unwrap_or(Value::Null),
"total_used": balance.get("total_used").cloned().unwrap_or(Value::Null),
"total_granted": balance.get("total_granted").cloned().unwrap_or(Value::Null),
"currency": balance.get("currency").cloned().unwrap_or_else(|| json!("USD")),
"plan_name": balance.get("plan_name").cloned().unwrap_or(Value::Null),
"query_config": balance.get("query_config").cloned().unwrap_or(Value::Null),
"extra": balance.get("extra").cloned().unwrap_or(Value::Null),
})
}
pub(crate) fn build_admin_provider_summary_value(
provider: &StoredProviderCatalogProvider,
endpoints: &[StoredProviderCatalogEndpoint],
@@ -212,6 +138,13 @@ pub(crate) fn build_admin_provider_summary_value(
.and_then(|cfg| cfg.get("simulated_cache_enabled"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let ops_quota_alert_enabled = provider_ops_config
.and_then(serde_json::Value::as_object)
.and_then(|cfg| cfg.get("quota_alert"))
.and_then(serde_json::Value::as_object)
.and_then(|cfg| cfg.get("enabled"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let billing_type = quota_snapshot
.map(|quota| quota.billing_type.clone())
.or_else(|| provider.billing_type.clone());
@@ -232,7 +165,6 @@ pub(crate) fn build_admin_provider_summary_value(
.and_then(|quota| quota.quota_expires_at_unix_secs)
.or(provider.quota_expires_at_unix_secs)
.and_then(unix_secs_to_rfc3339);
let key_balance_summary = latest_key_balance_summary(keys);
json!({
"id": provider.id.clone(),
@@ -271,8 +203,8 @@ pub(crate) fn build_admin_provider_summary_value(
"endpoint_health_details": endpoint_health_details,
"ops_configured": ops_configured,
"ops_architecture_id": ops_architecture_id,
"key_balance_summary": key_balance_summary,
"kiro_simulated_cache_enabled": kiro_simulated_cache_enabled,
"ops_quota_alert_enabled": ops_quota_alert_enabled,
"created_at": endpoint_timestamp_or_now(provider.created_at_unix_ms, now_unix_secs),
"updated_at": endpoint_timestamp_or_now(provider.updated_at_unix_secs, now_unix_secs),
})

View File

@@ -4,9 +4,9 @@ pub(crate) fn normalize_provider_type_input(value: &str) -> Result<String, Strin
let normalized = value.trim().to_ascii_lowercase();
match normalized.as_str() {
"custom" | "claude_code" | "kiro" | "codex" | "chatgpt_web" | "gemini_cli"
| "antigravity" | "vertex_ai" | "grok" => Ok(normalized),
| "antigravity" | "vertex_ai" | "grok" | "windsurf" => Ok(normalized),
_ => Err(
"provider_type 仅支持 custom / claude_code / kiro / codex / chatgpt_web / gemini_cli / antigravity / vertex_ai / grok"
"provider_type 仅支持 custom / claude_code / kiro / codex / chatgpt_web / gemini_cli / antigravity / vertex_ai / grok / windsurf"
.to_string(),
),
}

View File

@@ -177,21 +177,6 @@ impl<'a> AdminAppState<'a> {
self.app.update_provider_catalog_key(key).await
}
pub(crate) async fn update_provider_catalog_key_upstream_metadata(
&self,
key_id: &str,
upstream_metadata: Option<&serde_json::Value>,
updated_at_unix_secs: Option<u64>,
) -> Result<bool, GatewayError> {
self.app
.update_provider_catalog_key_upstream_metadata(
key_id,
upstream_metadata,
updated_at_unix_secs,
)
.await
}
pub(crate) async fn create_provider_catalog_key(
&self,
key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,

View File

@@ -49,7 +49,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::time::{SystemTime, UNIX_EPOCH};
use uuid::Uuid;
const ADMIN_SYSTEM_IMPORT_MAX_SIZE_BYTES: usize = 10 * 1024 * 1024;
const ADMIN_SYSTEM_IMPORT_MAX_SIZE_BYTES: usize = 500 * 1024 * 1024;
fn invalid_request(detail: impl Into<String>) -> (http::StatusCode, Value) {
(
@@ -173,6 +173,9 @@ fn normalize_import_endpoint_format(value: &str) -> Result<String, String> {
let normalized = match value.trim().to_ascii_lowercase().as_str() {
"openai:cli" => "openai:responses",
"openai:compact" => "openai:responses:compact",
"openai_image" | "images" | "image" | "/v1/images/generations" | "/v1/images/edits" => {
"openai:image"
}
"claude:chat" | "claude:cli" => "claude:messages",
"gemini:chat" | "gemini:cli" => "gemini:generate_content",
_ => value.trim(),
@@ -956,7 +959,7 @@ impl<'a> AdminAppState<'a> {
}
if request_body.len() > ADMIN_SYSTEM_DATA_IMPORT_MAX_SIZE_BYTES {
return Ok(Err(invalid_request("请求体大小不能超过 20MB")));
return Ok(Err(invalid_request("请求体大小不能超过 500MB")));
}
let root = match serde_json::from_slice::<Value>(request_body) {
@@ -1053,7 +1056,7 @@ impl<'a> AdminAppState<'a> {
)));
}
if request_body.len() > ADMIN_SYSTEM_IMPORT_MAX_SIZE_BYTES {
return Ok(Err(invalid_request("请求体大小不能超过 10MB")));
return Ok(Err(invalid_request("请求体大小不能超过 500MB")));
}
let parsed = routed!(parse_admin_system_config_import_request(request_body));
@@ -1993,7 +1996,7 @@ impl<'a> AdminAppState<'a> {
)));
}
if request_body.len() > ADMIN_SYSTEM_IMPORT_MAX_SIZE_BYTES {
return Ok(Err(invalid_request("请求体大小不能超过 10MB")));
return Ok(Err(invalid_request("请求体大小不能超过 500MB")));
}
let root = match serde_json::from_slice::<Value>(request_body) {
@@ -3062,6 +3065,10 @@ mod tests {
for (raw, expected) in [
("openai:cli", "openai:responses"),
("openai:compact", "openai:responses:compact"),
("openai_image", "openai:image"),
("images", "openai:image"),
("/v1/images/generations", "openai:image"),
("/v1/images/edits", "openai:image"),
("claude:chat", "claude:messages"),
("claude:cli", "claude:messages"),
("gemini:chat", "gemini:generate_content"),

View File

@@ -9,7 +9,7 @@ mod proxy_nodes;
mod templates;
const ADMIN_SYSTEM_DATA_EXPORT_VERSION: &str = "1.0";
const ADMIN_SYSTEM_DATA_IMPORT_MAX_SIZE_BYTES: usize = 20 * 1024 * 1024;
const ADMIN_SYSTEM_DATA_IMPORT_MAX_SIZE_BYTES: usize = 500 * 1024 * 1024;
impl<'a> AdminAppState<'a> {
pub(crate) async fn upsert_system_config_json_value(

View File

@@ -23,6 +23,7 @@ use crate::handlers::admin::system::shared::update::{
read_update_history, read_update_task_status, start_admin_system_rollback_task,
start_admin_system_update_task,
};
use crate::important_notification::build_important_notification_test_payload;
use crate::maintenance::{ManualUsageCleanupMode, ManualUsageCleanupOptions};
use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageCleanupTargets;
@@ -365,6 +366,16 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
));
}
if decision.route_kind.as_deref() == Some("important_notification_test")
&& request_method == http::Method::POST
&& request_path == "/api/admin/system/important-notification/test"
{
return Ok(Some(
Json(build_important_notification_test_payload(state, request_body).await?)
.into_response(),
));
}
if decision.route_kind.as_deref() == Some("cleanup") && request_method == http::Method::POST {
return Ok(Some(attach_admin_audit_response(
Json(build_admin_system_cleanup_payload(state).await?).into_response(),

View File

@@ -33,6 +33,21 @@ fn admin_system_config_default_value(key: &str) -> Option<serde_json::Value> {
admin_system_config_default_value_pure(key)
}
fn legacy_admin_system_config_fallback_key(normalized_key: &str) -> Option<&'static str> {
match normalized_key {
"module.server_chan_push.enabled" => {
Some("module.important_notification.server_chan_enabled")
}
"module.server_chan_push.send_key" => {
Some("module.important_notification.server_chan_send_key")
}
"module.server_chan_push.template" => {
Some("module.important_notification.server_chan_template")
}
_ => None,
}
}
pub(crate) fn build_admin_system_configs_payload(
entries: &[aether_data::repository::system::StoredSystemConfigEntry],
) -> serde_json::Value {
@@ -44,12 +59,14 @@ pub(crate) async fn build_admin_system_config_detail_payload(
requested_key: &str,
) -> Result<Result<serde_json::Value, (http::StatusCode, serde_json::Value)>, GatewayError> {
let requested_key = requested_key.trim();
let value = state
.read_system_config_json_value(&normalize_admin_system_config_key(requested_key))
.await?
.or_else(|| {
admin_system_config_default_value(&normalize_admin_system_config_key(requested_key))
});
let normalized_key = normalize_admin_system_config_key(requested_key);
let mut value = state.read_system_config_json_value(&normalized_key).await?;
if value.is_none() {
if let Some(legacy_key) = legacy_admin_system_config_fallback_key(&normalized_key) {
value = state.read_system_config_json_value(legacy_key).await?;
}
}
let value = value.or_else(|| admin_system_config_default_value(&normalized_key));
Ok(build_admin_system_config_detail_payload_pure(
requested_key,
value,

View File

@@ -1,5 +1,11 @@
use crate::bark_push::bark_push_configured;
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::shared::{module_available_from_env, system_config_bool};
use crate::important_notification::{
important_notification_configured, IMPORTANT_NOTIFICATION_ENABLED_KEY,
LEGACY_NOTIFICATION_EMAIL_ENABLED_KEY,
};
use crate::server_chan_push::server_chan_push_configured;
use crate::system_features::ENABLE_MODEL_DIRECTIVES_CONFIG_KEY;
use crate::GatewayError;
use aether_admin::system as admin_system_kernel;
@@ -68,17 +74,41 @@ pub(crate) const ADMIN_MODULE_DEFINITIONS: &[AdminModuleDefinition] = &[
admin_menu_order: 59,
},
AdminModuleDefinition {
name: "notification_email",
display_name: "异常通知",
description: "为 5xx 异常发送邮件通知,可在模块管理中启用或禁",
name: "important_notification",
display_name: "通知服务",
description: "统一管理通知项、模板和推送服务选择,供后台任务和用户通知使",
category: "integration",
env_key: "NOTIFICATION_EMAIL_AVAILABLE",
env_key: "IMPORTANT_NOTIFICATION_AVAILABLE",
default_available: true,
admin_route: None,
admin_menu_icon: Some("Mail"),
admin_menu_group: Some("system"),
admin_route: Some("/admin/notification-service"),
admin_menu_icon: Some("BellRing"),
admin_menu_group: None,
admin_menu_order: 58,
},
AdminModuleDefinition {
name: "server_chan_push",
display_name: "Server 酱推送",
description: "第三方推送服务,配置 Server 酱 Turbo SendKey 并测试微信推送",
category: "integration",
env_key: "SERVER_CHAN_PUSH_AVAILABLE",
default_available: true,
admin_route: Some("/admin/modules/server-chan"),
admin_menu_icon: Some("Send"),
admin_menu_group: Some("system"),
admin_menu_order: 59,
},
AdminModuleDefinition {
name: "bark_push",
display_name: "Bark 推送",
description: "第三方推送服务,配置 Bark Device Key 并测试 iOS 推送",
category: "integration",
env_key: "BARK_PUSH_AVAILABLE",
default_available: true,
admin_route: Some("/admin/modules/bark"),
admin_menu_icon: Some("Send"),
admin_menu_group: Some("system"),
admin_menu_order: 59,
},
AdminModuleDefinition {
name: "model_directives",
display_name: "模型后缀参数",
@@ -150,10 +180,17 @@ pub(crate) struct AdminModuleRuntimeState {
oauth_providers: Vec<aether_data::repository::auth_modules::StoredOAuthProviderModuleConfig>,
ldap_config: Option<aether_data::repository::auth_modules::StoredLdapModuleConfig>,
gemini_files_has_capable_key: bool,
smtp_configured: bool,
important_notification_configured: bool,
server_chan_push_configured: bool,
bark_push_configured: bool,
}
pub(crate) fn admin_module_by_name(name: &str) -> Option<&'static AdminModuleDefinition> {
let name = if name == "notification_email" {
"important_notification"
} else {
name
};
ADMIN_MODULE_DEFINITIONS
.iter()
.find(|module| module.name == name)
@@ -170,11 +207,22 @@ pub(crate) fn admin_module_name_from_enabled_path(request_path: &str) -> Option<
pub(crate) fn admin_module_enabled_config_key(module: &AdminModuleDefinition) -> String {
if module.name == "model_directives" {
ENABLE_MODEL_DIRECTIVES_CONFIG_KEY.to_string()
} else if module.name == "important_notification" {
IMPORTANT_NOTIFICATION_ENABLED_KEY.to_string()
} else {
format!("module.{}.enabled", module.name)
}
}
fn admin_module_available(module: &AdminModuleDefinition) -> bool {
if module.name == "important_notification" {
let legacy_default =
module_available_from_env("NOTIFICATION_EMAIL_AVAILABLE", module.default_available);
return module_available_from_env(module.env_key, legacy_default);
}
module_available_from_env(module.env_key, module.default_available)
}
pub(crate) fn oauth_module_config_is_valid(
providers: &[aether_data::repository::auth_modules::StoredOAuthProviderModuleConfig],
) -> bool {
@@ -221,28 +269,17 @@ pub(crate) async fn build_admin_module_runtime_state(
})
};
let smtp_host = state.read_system_config_json_value("smtp_host").await?;
let smtp_from_email = state
.read_system_config_json_value("smtp_from_email")
.await?;
let smtp_configured = smtp_host
.as_ref()
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.is_some()
&& smtp_from_email
.as_ref()
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.is_some();
let notification_configured = important_notification_configured(state.app()).await?;
let server_chan_configured = server_chan_push_configured(state.app()).await?;
let bark_configured = bark_push_configured(state.app()).await?;
Ok(AdminModuleRuntimeState {
oauth_providers,
ldap_config,
gemini_files_has_capable_key,
smtp_configured,
important_notification_configured: notification_configured,
server_chan_push_configured: server_chan_configured,
bark_push_configured: bark_configured,
})
}
@@ -255,7 +292,9 @@ pub(crate) fn build_admin_module_validation_result(
&runtime.oauth_providers,
runtime.ldap_config.as_ref(),
runtime.gemini_files_has_capable_key,
runtime.smtp_configured,
runtime.important_notification_configured,
runtime.server_chan_push_configured,
runtime.bark_push_configured,
)
}
@@ -274,12 +313,19 @@ pub(crate) async fn build_admin_module_status_payload(
module: &AdminModuleDefinition,
runtime: &AdminModuleRuntimeState,
) -> Result<serde_json::Value, GatewayError> {
let available = module_available_from_env(module.env_key, module.default_available);
let available = admin_module_available(module);
let enabled = if available {
let enabled = state
let enabled_value = state
.read_system_config_json_value(&admin_module_enabled_config_key(module))
.await?;
system_config_bool(enabled.as_ref(), false)
let enabled_value = if module.name == "important_notification" && enabled_value.is_none() {
state
.read_system_config_json_value(LEGACY_NOTIFICATION_EMAIL_ENABLED_KEY)
.await?
} else {
enabled_value
};
system_config_bool(enabled_value.as_ref(), false)
} else {
false
};

View File

@@ -1,14 +1,10 @@
use crate::email_delivery::{probe_smtp_connection, system_config_u16, SmtpDeliveryConfig};
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::shared::{system_config_bool, system_config_string};
use crate::GatewayError;
use axum::body::Bytes;
use base64::Engine;
use serde::Deserialize;
use serde_json::json;
use std::io::{BufRead, Write};
use std::time::Duration;
const SMTP_TIMEOUT_SECS: u64 = 30;
#[derive(Debug, Default, Deserialize)]
struct AdminSmtpTestRequest {
@@ -53,12 +49,12 @@ pub(crate) async fn build_admin_smtp_test_payload(
}));
}
let result = tokio::task::spawn_blocking(move || test_smtp_connection_blocking(config))
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let result = probe_smtp_connection(config.into_delivery_config()).await;
Ok(match result {
Ok(()) => json!({ "success": true, "message": "SMTP 连接测试成功" }),
Err(error) => json!({ "success": false, "message": translate_smtp_error(&error) }),
Err(error) => {
json!({ "success": false, "message": translate_smtp_error(&smtp_gateway_error_message(&error)) })
}
})
}
@@ -94,8 +90,8 @@ async fn resolve_admin_smtp_config(
port: request
.smtp_port
.as_ref()
.map(|value| system_config_u16(value, 587))
.unwrap_or_else(|| system_config_u16_opt(smtp_port.as_ref(), 587)),
.map(|value| system_config_u16(Some(value), 587))
.unwrap_or_else(|| system_config_u16(smtp_port.as_ref(), 587)),
user: request
.smtp_user
.as_ref()
@@ -130,6 +126,21 @@ async fn resolve_admin_smtp_config(
})
}
impl ResolvedSmtpConfig {
fn into_delivery_config(self) -> SmtpDeliveryConfig {
SmtpDeliveryConfig {
host: self.host.unwrap_or_default(),
port: self.port,
user: self.user,
password: self.password,
use_tls: self.use_tls,
use_ssl: self.use_ssl,
from_email: self.from_email.unwrap_or_default(),
from_name: self.from_name,
}
}
}
fn missing_smtp_fields(config: &ResolvedSmtpConfig) -> Vec<&'static str> {
let mut fields = Vec::new();
if config
@@ -171,178 +182,13 @@ fn missing_smtp_fields(config: &ResolvedSmtpConfig) -> Vec<&'static str> {
fields
}
fn system_config_u16_opt(value: Option<&serde_json::Value>, default: u16) -> u16 {
value
.map(|value| system_config_u16(value, default))
.unwrap_or(default)
}
fn system_config_u16(value: &serde_json::Value, default: u16) -> u16 {
match value {
serde_json::Value::Number(value) => value
.as_u64()
.and_then(|value| u16::try_from(value).ok())
.unwrap_or(default),
serde_json::Value::String(value) => value.trim().parse::<u16>().unwrap_or(default),
_ => default,
fn smtp_gateway_error_message(error: &GatewayError) -> String {
match error {
GatewayError::Internal(message) => message.clone(),
_ => format!("{error:?}"),
}
}
fn build_tls_config() -> std::sync::Arc<rustls::ClientConfig> {
let _ = rustls::crypto::ring::default_provider().install_default();
let root_store =
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
std::sync::Arc::new(
rustls::ClientConfig::builder()
.with_root_certificates(root_store)
.with_no_client_auth(),
)
}
fn resolve_server_name(host: &str) -> Result<rustls::pki_types::ServerName<'static>, String> {
let host = host.trim().trim_start_matches('[').trim_end_matches(']');
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
return Ok(rustls::pki_types::ServerName::from(ip));
}
rustls::pki_types::ServerName::try_from(host.to_string()).map_err(|err| err.to_string())
}
fn connect_tcp_stream(config: &ResolvedSmtpConfig) -> Result<std::net::TcpStream, String> {
let host = config.host.as_deref().unwrap_or_default();
let stream =
std::net::TcpStream::connect((host, config.port)).map_err(|err| err.to_string())?;
stream
.set_read_timeout(Some(Duration::from_secs(SMTP_TIMEOUT_SECS)))
.map_err(|err| err.to_string())?;
stream
.set_write_timeout(Some(Duration::from_secs(SMTP_TIMEOUT_SECS)))
.map_err(|err| err.to_string())?;
Ok(stream)
}
fn wrap_tls_stream(
stream: std::net::TcpStream,
host: &str,
) -> Result<rustls::StreamOwned<rustls::ClientConnection, std::net::TcpStream>, String> {
let server_name = resolve_server_name(host)?;
let connection = rustls::ClientConnection::new(build_tls_config(), server_name)
.map_err(|err| err.to_string())?;
Ok(rustls::StreamOwned::new(connection, stream))
}
fn smtp_read_response<T: BufRead>(reader: &mut T) -> Result<(u16, String), String> {
let mut message = String::new();
let code = loop {
let mut line = String::new();
let bytes = reader.read_line(&mut line).map_err(|err| err.to_string())?;
if bytes == 0 {
return Err("smtp connection closed unexpectedly".to_string());
}
let trimmed = line.trim_end_matches(['\r', '\n']).to_string();
if trimmed.len() < 3 {
return Err("invalid smtp response".to_string());
}
let parsed_code = trimmed[..3].parse::<u16>().map_err(|err| err.to_string())?;
let continuation = trimmed.as_bytes().get(3).copied() == Some(b'-');
if !message.is_empty() {
message.push('\n');
}
message.push_str(&trimmed);
if !continuation {
break parsed_code;
}
};
Ok((code, message))
}
fn smtp_expect<T: BufRead>(reader: &mut T, allowed_codes: &[u16]) -> Result<String, String> {
let (code, message) = smtp_read_response(reader)?;
if allowed_codes.contains(&code) {
return Ok(message);
}
Err(format!("unexpected smtp response {code}: {message}"))
}
fn smtp_write_line<T: Write>(writer: &mut T, line: &str) -> Result<(), String> {
writer
.write_all(line.as_bytes())
.map_err(|err| err.to_string())?;
writer.write_all(b"\r\n").map_err(|err| err.to_string())?;
writer.flush().map_err(|err| err.to_string())
}
fn smtp_send_command<S: std::io::Read + Write>(
reader: &mut std::io::BufReader<S>,
command: &str,
allowed_codes: &[u16],
) -> Result<String, String> {
smtp_write_line(reader.get_mut(), command)?;
smtp_expect(reader, allowed_codes)
}
fn smtp_authenticate<S: std::io::Read + Write>(
reader: &mut std::io::BufReader<S>,
config: &ResolvedSmtpConfig,
) -> Result<(), String> {
let Some(username) = config
.user
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(());
};
let password = config.password.as_deref().unwrap_or_default();
smtp_send_command(reader, "AUTH LOGIN", &[334])?;
smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(username.as_bytes()),
&[334],
)?;
smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(password.as_bytes()),
&[235],
)?;
Ok(())
}
fn smtp_probe<S: std::io::Read + Write>(
reader: &mut std::io::BufReader<S>,
config: &ResolvedSmtpConfig,
) -> Result<(), String> {
smtp_send_command(reader, "EHLO aether.local", &[250])?;
smtp_authenticate(reader, config)?;
let _ = smtp_send_command(reader, "QUIT", &[221]);
Ok(())
}
fn test_smtp_connection_blocking(config: ResolvedSmtpConfig) -> Result<(), String> {
if config.use_ssl {
let stream = connect_tcp_stream(&config)?;
let tls_stream = wrap_tls_stream(stream, config.host.as_deref().unwrap_or_default())?;
let mut reader = std::io::BufReader::new(tls_stream);
smtp_expect(&mut reader, &[220])?;
return smtp_probe(&mut reader, &config);
}
let stream = connect_tcp_stream(&config)?;
let mut reader = std::io::BufReader::new(stream);
smtp_expect(&mut reader, &[220])?;
smtp_send_command(&mut reader, "EHLO aether.local", &[250])?;
if config.use_tls {
smtp_send_command(&mut reader, "STARTTLS", &[220])?;
let stream = reader.into_inner();
let tls_stream = wrap_tls_stream(stream, config.host.as_deref().unwrap_or_default())?;
let mut reader = std::io::BufReader::new(tls_stream);
return smtp_probe(&mut reader, &config);
}
smtp_authenticate(&mut reader, &config)?;
let _ = smtp_send_command(&mut reader, "QUIT", &[221]);
Ok(())
}
fn translate_smtp_error(error: &str) -> String {
let error_lower = error.to_ascii_lowercase();

View File

@@ -1,11 +1,11 @@
use super::{
decrypt_catalog_secret_with_fallbacks, escape_admin_email_template_html, json,
read_admin_email_template_payload, render_admin_email_template_html, system_config_bool,
system_config_string, system_config_u16, AppState, GatewayError,
escape_admin_email_template_html, json, read_admin_email_template_payload,
render_admin_email_template_html, system_config_string, AppState, GatewayError,
AUTH_EMAIL_VERIFICATION_PREFIX, AUTH_EMAIL_VERIFIED_PREFIX, AUTH_EMAIL_VERIFIED_TTL_SECS,
AUTH_SMTP_TIMEOUT_SECS,
};
use base64::Engine;
use crate::email_delivery::{
read_smtp_delivery_config, send_smtp_email, ComposedEmail, SmtpDeliveryConfig,
};
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(super) struct StoredAuthEmailVerificationCode {
@@ -13,25 +13,8 @@ pub(super) struct StoredAuthEmailVerificationCode {
pub(super) created_at: String,
}
#[derive(Debug, Clone)]
pub(super) struct AuthSmtpConfig {
pub(super) host: String,
pub(super) port: u16,
pub(super) user: Option<String>,
pub(super) password: Option<String>,
pub(super) use_tls: bool,
pub(super) use_ssl: bool,
pub(super) from_email: String,
pub(super) from_name: String,
}
#[derive(Debug, Clone)]
pub(super) struct AuthComposedEmail {
pub(super) to_email: String,
pub(super) subject: String,
pub(super) html_body: String,
pub(super) text_body: String,
}
pub(super) type AuthSmtpConfig = SmtpDeliveryConfig;
pub(super) type AuthComposedEmail = ComposedEmail;
pub(super) fn auth_email_verification_key(email: &str) -> String {
format!("{AUTH_EMAIL_VERIFICATION_PREFIX}{email}")
@@ -84,25 +67,6 @@ fn render_auth_template_string(
Ok(rendered)
}
fn auth_encode_mime_header(value: &str) -> String {
if value.is_ascii() {
return value.to_string();
}
format!(
"=?UTF-8?B?{}?=",
base64::engine::general_purpose::STANDARD.encode(value.as_bytes())
)
}
fn auth_wrap_base64(value: &str) -> String {
let mut wrapped = String::new();
for chunk in value.as_bytes().chunks(76) {
wrapped.push_str(std::str::from_utf8(chunk).unwrap_or_default());
wrapped.push_str("\r\n");
}
wrapped
}
fn auth_build_verification_text_body(
app_name: &str,
email: &str,
@@ -114,244 +78,6 @@ fn auth_build_verification_text_body(
)
}
fn auth_build_tls_config() -> std::sync::Arc<rustls::ClientConfig> {
let _ = rustls::crypto::ring::default_provider().install_default();
let root_store =
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
let config = rustls::ClientConfig::builder()
.with_root_certificates(root_store)
.with_no_client_auth();
std::sync::Arc::new(config)
}
fn auth_resolve_server_name(
host: &str,
) -> Result<rustls::pki_types::ServerName<'static>, GatewayError> {
let host = host.trim().trim_start_matches('[').trim_end_matches(']');
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
return Ok(rustls::pki_types::ServerName::from(ip));
}
rustls::pki_types::ServerName::try_from(host.to_string())
.map_err(|err| GatewayError::Internal(err.to_string()))
}
fn auth_connect_tcp_stream(config: &AuthSmtpConfig) -> Result<std::net::TcpStream, GatewayError> {
let stream = std::net::TcpStream::connect((config.host.as_str(), config.port))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
stream
.set_read_timeout(Some(std::time::Duration::from_secs(AUTH_SMTP_TIMEOUT_SECS)))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
stream
.set_write_timeout(Some(std::time::Duration::from_secs(AUTH_SMTP_TIMEOUT_SECS)))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(stream)
}
fn auth_wrap_tls_stream(
stream: std::net::TcpStream,
host: &str,
) -> Result<rustls::StreamOwned<rustls::ClientConnection, std::net::TcpStream>, GatewayError> {
let server_name = auth_resolve_server_name(host)?;
let connection = rustls::ClientConnection::new(auth_build_tls_config(), server_name)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(rustls::StreamOwned::new(connection, stream))
}
fn auth_smtp_read_response<T: std::io::BufRead>(
reader: &mut T,
) -> Result<(u16, String), GatewayError> {
let mut message = String::new();
let code = loop {
let parsed_code;
let continuation;
let trimmed;
{
let mut line = String::new();
let bytes = reader
.read_line(&mut line)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if bytes == 0 {
return Err(GatewayError::Internal(
"smtp connection closed unexpectedly".to_string(),
));
}
trimmed = line.trim_end_matches(['\r', '\n']).to_string();
if trimmed.len() < 3 {
return Err(GatewayError::Internal("invalid smtp response".to_string()));
}
parsed_code = trimmed[..3]
.parse::<u16>()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
continuation = trimmed.as_bytes().get(3).copied() == Some(b'-');
}
if !message.is_empty() {
message.push('\n');
}
message.push_str(&trimmed);
if !continuation {
break parsed_code;
}
};
Ok((code, message))
}
fn auth_smtp_expect<T: std::io::BufRead>(
reader: &mut T,
allowed_codes: &[u16],
) -> Result<String, GatewayError> {
let (code, message) = auth_smtp_read_response(reader)?;
if allowed_codes.contains(&code) {
return Ok(message);
}
Err(GatewayError::Internal(format!(
"unexpected smtp response {code}: {message}"
)))
}
fn auth_smtp_write_line<T: std::io::Write>(writer: &mut T, line: &str) -> Result<(), GatewayError> {
writer
.write_all(line.as_bytes())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
writer
.write_all(b"\r\n")
.map_err(|err| GatewayError::Internal(err.to_string()))?;
writer
.flush()
.map_err(|err| GatewayError::Internal(err.to_string()))
}
fn auth_smtp_send_command<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
command: &str,
allowed_codes: &[u16],
) -> Result<String, GatewayError> {
auth_smtp_write_line(reader.get_mut(), command)?;
auth_smtp_expect(reader, allowed_codes)
}
fn auth_build_email_message(config: &AuthSmtpConfig, email: &AuthComposedEmail) -> String {
let boundary = format!("aether-{}", uuid::Uuid::new_v4().simple());
let text_body = auth_wrap_base64(
&base64::engine::general_purpose::STANDARD.encode(email.text_body.as_bytes()),
);
let html_body = auth_wrap_base64(
&base64::engine::general_purpose::STANDARD.encode(email.html_body.as_bytes()),
);
let from_header = if config.from_name.trim().is_empty() {
format!("<{}>", config.from_email)
} else {
format!(
"{} <{}>",
auth_encode_mime_header(config.from_name.trim()),
config.from_email
)
};
format!(
"From: {from_header}\r\nTo: <{to_email}>\r\nSubject: {subject}\r\nMIME-Version: 1.0\r\nContent-Type: multipart/alternative; boundary=\"{boundary}\"\r\n\r\n--{boundary}\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{text_body}--{boundary}\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{html_body}--{boundary}--\r\n",
to_email = email.to_email,
subject = auth_encode_mime_header(&email.subject),
)
}
fn auth_smtp_authenticate<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &AuthSmtpConfig,
) -> Result<(), GatewayError> {
let Some(username) = config
.user
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(());
};
let password = config.password.as_deref().unwrap_or("");
auth_smtp_send_command(reader, "AUTH LOGIN", &[334])?;
auth_smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(username.as_bytes()),
&[334],
)?;
auth_smtp_send_command(
reader,
&base64::engine::general_purpose::STANDARD.encode(password.as_bytes()),
&[235],
)?;
Ok(())
}
fn auth_smtp_deliver_message<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &AuthSmtpConfig,
email: &AuthComposedEmail,
) -> Result<(), GatewayError> {
auth_smtp_send_command(
reader,
&format!("MAIL FROM:<{}>", config.from_email),
&[250],
)?;
auth_smtp_send_command(
reader,
&format!("RCPT TO:<{}>", email.to_email),
&[250, 251],
)?;
auth_smtp_send_command(reader, "DATA", &[354])?;
let message = auth_build_email_message(config, email);
reader
.get_mut()
.write_all(message.as_bytes())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
reader
.get_mut()
.write_all(b"\r\n.\r\n")
.map_err(|err| GatewayError::Internal(err.to_string()))?;
reader
.get_mut()
.flush()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let _ = auth_smtp_expect(reader, &[250])?;
let _ = auth_smtp_send_command(reader, "QUIT", &[221]);
Ok(())
}
fn auth_smtp_send_message<S: std::io::Read + std::io::Write>(
reader: &mut std::io::BufReader<S>,
config: &AuthSmtpConfig,
email: &AuthComposedEmail,
) -> Result<(), GatewayError> {
auth_smtp_send_command(reader, "EHLO aether.local", &[250])?;
auth_smtp_authenticate(reader, config)?;
auth_smtp_deliver_message(reader, config, email)
}
fn send_auth_email_blocking(
config: AuthSmtpConfig,
email: AuthComposedEmail,
) -> Result<(), GatewayError> {
if config.use_ssl {
let stream = auth_connect_tcp_stream(&config)?;
let tls_stream = auth_wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
let _ = auth_smtp_expect(&mut reader, &[220])?;
return auth_smtp_send_message(&mut reader, &config, &email);
}
let stream = auth_connect_tcp_stream(&config)?;
let mut reader = std::io::BufReader::new(stream);
let _ = auth_smtp_expect(&mut reader, &[220])?;
let _ = auth_smtp_send_command(&mut reader, "EHLO aether.local", &[250])?;
if config.use_tls {
let _ = auth_smtp_send_command(&mut reader, "STARTTLS", &[220])?;
let stream = reader.into_inner();
let tls_stream = auth_wrap_tls_stream(stream, &config.host)?;
let mut reader = std::io::BufReader::new(tls_stream);
return auth_smtp_send_message(&mut reader, &config, &email);
}
auth_smtp_authenticate(&mut reader, &config)?;
auth_smtp_deliver_message(&mut reader, &config, &email)
}
pub(super) async fn read_auth_email_verification_code(
state: &AppState,
email: &str,
@@ -423,40 +149,7 @@ pub(super) async fn store_auth_email_verification_code(
pub(super) async fn read_auth_smtp_config(
state: &AppState,
) -> Result<Option<AuthSmtpConfig>, GatewayError> {
let smtp_host = state.read_system_config_json_value("smtp_host").await?;
let smtp_from_email = state
.read_system_config_json_value("smtp_from_email")
.await?;
let Some(host) = system_config_string(smtp_host.as_ref()) else {
return Ok(None);
};
let Some(from_email) = system_config_string(smtp_from_email.as_ref()) else {
return Ok(None);
};
let smtp_port = state.read_system_config_json_value("smtp_port").await?;
let smtp_user = state.read_system_config_json_value("smtp_user").await?;
let smtp_password = state.read_system_config_json_value("smtp_password").await?;
let smtp_use_tls = state.read_system_config_json_value("smtp_use_tls").await?;
let smtp_use_ssl = state.read_system_config_json_value("smtp_use_ssl").await?;
let smtp_from_name = state
.read_system_config_json_value("smtp_from_name")
.await?;
let password = system_config_string(smtp_password.as_ref()).map(|value| {
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
});
Ok(Some(AuthSmtpConfig {
host,
port: system_config_u16(smtp_port.as_ref(), 587),
user: system_config_string(smtp_user.as_ref()),
password,
use_tls: system_config_bool(smtp_use_tls.as_ref(), true),
use_ssl: system_config_bool(smtp_use_ssl.as_ref(), false),
from_email,
from_name: system_config_string(smtp_from_name.as_ref())
.unwrap_or_else(|| "Aether".to_string()),
}))
read_smtp_delivery_config(state).await
}
pub(super) async fn auth_email_app_name(state: &AppState) -> Result<String, GatewayError> {
@@ -516,27 +209,20 @@ pub(super) async fn send_auth_email(
if record_auth_email_delivery_for_tests(
state,
json!({
"to_email": email.to_email,
"subject": email.subject,
"html_body": email.html_body,
"text_body": email.text_body,
"to_email": email.to_email.clone(),
"subject": email.subject.clone(),
"html_body": email.html_body.clone(),
"text_body": email.text_body.clone(),
}),
) {
return Ok(());
}
tokio::task::spawn_blocking(move || send_auth_email_blocking(config, email))
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
send_smtp_email(config, email).await
}
pub(super) async fn auth_registration_email_configured(
state: &AppState,
) -> Result<bool, GatewayError> {
let smtp_host = state.read_system_config_json_value("smtp_host").await?;
let smtp_from_email = state
.read_system_config_json_value("smtp_from_email")
.await?;
Ok(system_config_string(smtp_host.as_ref()).is_some()
&& system_config_string(smtp_from_email.as_ref()).is_some())
Ok(read_smtp_delivery_config(state).await?.is_some())
}

View File

@@ -121,7 +121,6 @@ pub(super) const AUTH_REFRESH_TOKEN_EXPIRATION_DAYS: i64 = 7;
pub(super) const AUTH_EMAIL_VERIFICATION_PREFIX: &str = "email:verification:";
pub(super) const AUTH_EMAIL_VERIFIED_PREFIX: &str = "email:verified:";
pub(super) const AUTH_EMAIL_VERIFIED_TTL_SECS: u64 = 3600;
pub(super) const AUTH_SMTP_TIMEOUT_SECS: u64 = 30;
pub(crate) fn build_auth_json_response(
status: http::StatusCode,

View File

@@ -7,7 +7,9 @@ use axum::{
use serde::Deserialize;
use serde_json::json;
use crate::handlers::shared::{deserialize_optional_json_patch, normalize_feature_settings};
use crate::handlers::shared::{
deserialize_optional_json_patch, normalize_user_self_feature_settings_update,
};
use super::{
auth_password_policy_level, build_auth_error_response, resolve_authenticated_local_user,
@@ -65,12 +67,24 @@ pub(super) async fn handle_users_me_detail_put(
let email = normalize_users_me_optional_non_empty_string(payload.email);
let username = normalize_users_me_optional_non_empty_string(payload.username);
let feature_settings = match payload.feature_settings {
Some(value) => match normalize_feature_settings(value) {
Ok(value) => Some(value),
Err(detail) => {
return build_auth_error_response(http::StatusCode::BAD_REQUEST, detail, false);
Some(value) => {
let current = match state.read_user_feature_settings(&auth.user.id).await {
Ok(value) => value,
Err(err) => {
return build_auth_error_response(
http::StatusCode::INTERNAL_SERVER_ERROR,
format!("user feature settings lookup failed: {err:?}"),
false,
)
}
};
match normalize_user_self_feature_settings_update(value, current) {
Ok(value) => Some(value),
Err(detail) => {
return build_auth_error_response(http::StatusCode::BAD_REQUEST, detail, false);
}
}
},
}
None => None,
};

View File

@@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet};
use aether_ai_serving::UPSTREAM_IS_STREAM_KEY;
use aether_billing::{
normalize_input_tokens_for_billing, normalize_total_input_context_for_cache_hit_rate,
};
@@ -314,7 +315,7 @@ fn users_me_usage_upstream_is_stream(item: &StoredRequestUsageAudit) -> bool {
item.request_metadata
.as_ref()
.and_then(serde_json::Value::as_object)
.and_then(|metadata| metadata.get("upstream_is_stream"))
.and_then(|metadata| metadata.get(UPSTREAM_IS_STREAM_KEY))
.and_then(serde_json::Value::as_bool)
.or_else(|| users_me_usage_headers_stream_flag(item.response_headers.as_ref()))
.or_else(|| users_me_usage_infer_upstream_stream_from_captured_bodies(item))

View File

@@ -21,7 +21,6 @@ const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
const OAUTH_REQUEST_FAILED_PREFIX: &str = "[REQUEST_FAILED] ";
const BALANCE_QUERY_SECRET_CIPHERTEXT_KEY: &str = "secret_ciphertext";
pub(crate) fn provider_catalog_key_supports_format(
key: &StoredProviderCatalogKey,
@@ -174,31 +173,6 @@ pub(crate) fn parse_catalog_auth_config_json(
.cloned()
}
fn sanitized_admin_upstream_metadata(upstream_metadata: Option<&Value>) -> Value {
let Some(mut metadata) = upstream_metadata.cloned() else {
return Value::Null;
};
let Some(balance_query) = metadata
.as_object_mut()
.and_then(|metadata| metadata.get_mut("balance_query"))
.and_then(Value::as_object_mut)
else {
return metadata;
};
let has_saved_secret = balance_query
.remove(BALANCE_QUERY_SECRET_CIPHERTEXT_KEY)
.is_some();
if has_saved_secret {
let query_config = balance_query
.entry("query_config".to_string())
.or_insert_with(|| Value::Object(Map::new()));
if let Some(query_config) = query_config.as_object_mut() {
query_config.insert("has_saved_secret".to_string(), Value::Bool(true));
}
}
metadata
}
pub(crate) fn default_provider_key_status_snapshot() -> serde_json::Value {
json!({
"oauth": {
@@ -1112,6 +1086,278 @@ fn build_chatgpt_web_quota_status_snapshot(
}))
}
fn windsurf_percent_quota_window_snapshot(
metadata: &Map<String, Value>,
code: &str,
label: &str,
remaining_percent_key: &str,
reset_at_key: &str,
observed_at_unix_secs: Option<u64>,
) -> Option<Value> {
let remaining_percent = metadata
.get(remaining_percent_key)
.and_then(admin_provider_quota_pure::coerce_json_f64);
let reset_at = provider_quota_timestamp_unix_secs(metadata.get(reset_at_key));
if remaining_percent.is_none() && reset_at.is_none() {
return None;
}
let remaining_ratio = remaining_percent.map(|value| (value / 100.0).clamp(0.0, 1.0));
let used_ratio = remaining_ratio.map(|value| (1.0 - value).clamp(0.0, 1.0));
let reset_seconds = quota_window_reset_seconds(observed_at_unix_secs, reset_at);
Some(json!({
"code": code,
"label": label,
"scope": "account",
"unit": "percent",
"used_ratio": used_ratio,
"remaining_ratio": remaining_ratio,
"reset_at": reset_at,
"reset_seconds": reset_seconds,
"is_exhausted": remaining_ratio.map(|value| value <= 1e-6),
}))
}
fn windsurf_count_quota_window_snapshot(
metadata: &Map<String, Value>,
code: &str,
label: &str,
used_key: &str,
limit_key: &str,
remaining_key: &str,
) -> Option<Value> {
let used = metadata
.get(used_key)
.and_then(admin_provider_quota_pure::coerce_json_f64);
let limit = metadata
.get(limit_key)
.and_then(admin_provider_quota_pure::coerce_json_f64);
let remaining = metadata
.get(remaining_key)
.and_then(admin_provider_quota_pure::coerce_json_f64)
.or_else(|| limit.zip(used).map(|(limit, used)| (limit - used).max(0.0)));
if used.is_none() && limit.is_none() && remaining.is_none() {
return None;
}
let used_ratio = used
.zip(limit)
.and_then(|(used, limit)| (limit > 0.0).then_some((used / limit).clamp(0.0, 1.0)));
let remaining_ratio = remaining.zip(limit).and_then(|(remaining, limit)| {
(limit > 0.0).then_some((remaining / limit).clamp(0.0, 1.0))
});
Some(json!({
"code": code,
"label": label,
"scope": "account",
"unit": "count",
"used_ratio": used_ratio,
"remaining_ratio": remaining_ratio,
"used_value": used,
"remaining_value": remaining,
"limit_value": limit,
"is_exhausted": remaining.is_some_and(|value| value <= 0.0),
}))
}
fn build_windsurf_quota_status_snapshot(
upstream_metadata: Option<&Value>,
source: &str,
) -> Option<Value> {
let metadata = provider_quota_metadata_bucket(upstream_metadata, "windsurf")?;
let observed_at_unix_secs = provider_quota_timestamp_unix_secs(metadata.get("updated_at"));
let plan_type = metadata
.get("plan_name")
.or_else(|| metadata.get("plan_type"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let rate_limit = metadata
.get("rate_limit")
.cloned()
.filter(|value| !value.is_null());
let last_error = metadata
.get("last_error")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let banned = metadata
.get("banned")
.or_else(|| metadata.get("is_banned"))
.and_then(admin_provider_quota_pure::coerce_json_bool)
== Some(true);
let quarantined = metadata
.get("quarantined")
.or_else(|| metadata.get("is_quarantined"))
.and_then(admin_provider_quota_pure::coerce_json_bool)
== Some(true);
let mut windows = [
windsurf_percent_quota_window_snapshot(
metadata,
"daily",
"",
"daily_remaining_percent",
"daily_reset_at",
observed_at_unix_secs,
),
windsurf_percent_quota_window_snapshot(
metadata,
"weekly",
"",
"weekly_remaining_percent",
"weekly_reset_at",
observed_at_unix_secs,
),
windsurf_count_quota_window_snapshot(
metadata,
"prompt",
"Prompt",
"prompt_used",
"prompt_limit",
"prompt_remaining",
),
windsurf_count_quota_window_snapshot(
metadata,
"flex",
"Flex",
"flex_used",
"flex_limit",
"flex_remaining",
),
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
let mut rate_limit_cooling = false;
let mut rate_limit_reset_seconds = None;
let mut rate_limit_reason = None::<String>;
if let Some(rate_limit) = rate_limit.as_ref() {
if let Some(rate_limit_object) = rate_limit.as_object() {
let retry_after_ms = rate_limit_object
.get("retry_after_ms")
.or_else(|| rate_limit_object.get("retryAfterMs"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
.filter(|value| *value > 0);
if let Some(retry_after_ms) = retry_after_ms {
rate_limit_cooling = true;
rate_limit_reset_seconds = Some(retry_after_ms.saturating_add(999) / 1000);
rate_limit_reason = rate_limit_object
.get("message")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
windows.push(json!({
"code": "rate_limit",
"label": "速率",
"scope": "account",
"unit": "count",
"is_exhausted": false,
"reset_seconds": rate_limit_reset_seconds,
}));
}
}
}
let allowed_models_count = metadata
.get("allowed_models_count")
.or_else(|| metadata.get("models_count"))
.and_then(admin_provider_quota_pure::coerce_json_u64);
if windows.is_empty()
&& plan_type.is_none()
&& observed_at_unix_secs.is_none()
&& rate_limit.is_none()
&& allowed_models_count.is_none()
&& !banned
&& !quarantined
{
return None;
}
let usage_ratio = quota_windows_usage_ratio(&windows);
let reset_seconds = if rate_limit_cooling {
rate_limit_reset_seconds.or_else(|| quota_windows_min_reset_seconds(&windows))
} else {
quota_windows_min_reset_seconds(&windows)
};
let reset_at = if rate_limit_cooling {
None
} else {
quota_windows_min_reset_at(&windows)
};
let exhausted_by_window = windows.iter().filter_map(Value::as_object).any(|window| {
window
.get("code")
.and_then(Value::as_str)
.is_some_and(|code| {
code.eq_ignore_ascii_case("daily") || code.eq_ignore_ascii_case("weekly")
})
&& window
.get("is_exhausted")
.and_then(admin_provider_quota_pure::coerce_json_bool)
.unwrap_or(false)
});
let exhausted = banned || quarantined || exhausted_by_window;
let (code, label, reason) = if banned {
(
"banned",
Some("账号已封禁"),
last_error
.clone()
.or_else(|| Some("账号被 Windsurf 标记为不可用".to_string())),
)
} else if quarantined {
(
"quarantined",
Some("账号隔离中"),
last_error
.clone()
.or_else(|| Some("账号处于隔离状态".to_string())),
)
} else if rate_limit_cooling {
(
"cooldown",
Some("冷却中"),
last_error.clone().or(rate_limit_reason),
)
} else if exhausted {
(
"exhausted",
Some("额度耗尽"),
Some("额度窗口已耗尽".to_string()),
)
} else {
("ok", None, last_error)
};
Some(json!({
"version": 2,
"provider_type": "windsurf",
"code": code,
"label": label,
"reason": reason,
"freshness": "fresh",
"source": source,
"observed_at": observed_at_unix_secs,
"exhausted": exhausted,
"usage_ratio": usage_ratio,
"updated_at": observed_at_unix_secs,
"reset_at": reset_at,
"reset_seconds": reset_seconds,
"plan_type": plan_type,
"allowed_models_count": allowed_models_count,
"rate_limit": rate_limit.unwrap_or(Value::Null),
"windows": windows,
}))
}
fn build_antigravity_quota_status_snapshot(
upstream_metadata: Option<&Value>,
source: &str,
@@ -1359,6 +1605,7 @@ pub(crate) fn sync_provider_key_quota_status_snapshot(
"codex" => build_codex_quota_status_snapshot(upstream_metadata, source),
"kiro" => build_kiro_quota_status_snapshot(upstream_metadata, source),
"chatgpt_web" => build_chatgpt_web_quota_status_snapshot(upstream_metadata, source),
"windsurf" => build_windsurf_quota_status_snapshot(upstream_metadata, source),
"antigravity" => build_antigravity_quota_status_snapshot(upstream_metadata, source),
"grok" => build_grok_quota_status_snapshot(upstream_metadata, source),
"gemini_cli" => build_gemini_cli_quota_status_snapshot(upstream_metadata, source),
@@ -1395,6 +1642,12 @@ fn quota_snapshot_has_materialized_data(
return false;
}
if normalized_provider_type == "windsurf"
&& windsurf_quota_snapshot_has_stale_cooldown(quota_snapshot)
{
return false;
}
if quota_snapshot
.get("windows")
.and_then(Value::as_array)
@@ -1420,6 +1673,61 @@ fn quota_snapshot_has_materialized_data(
})
}
fn windsurf_quota_snapshot_has_stale_cooldown(quota_snapshot: &Map<String, Value>) -> bool {
let code = quota_snapshot
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !code.eq_ignore_ascii_case("cooldown") {
return false;
}
let rate_limit = quota_snapshot.get("rate_limit").and_then(Value::as_object);
let retry_after_ms = rate_limit
.and_then(|rate_limit| {
rate_limit
.get("retry_after_ms")
.or_else(|| rate_limit.get("retryAfterMs"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
})
.unwrap_or(0);
if retry_after_ms > 0 {
return false;
}
let has_positive_rate_limit_reset = quota_snapshot
.get("windows")
.and_then(Value::as_array)
.is_some_and(|windows| {
windows.iter().filter_map(Value::as_object).any(|window| {
window
.get("code")
.and_then(Value::as_str)
.is_some_and(|code| code.eq_ignore_ascii_case("rate_limit"))
&& window
.get("reset_seconds")
.or_else(|| window.get("reset_at"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
.is_some_and(|value| value > 0)
})
});
if has_positive_rate_limit_reset {
return false;
}
let exhausted = quota_snapshot
.get("exhausted")
.and_then(admin_provider_quota_pure::coerce_json_bool)
.unwrap_or(false);
let has_capacity = rate_limit
.and_then(|rate_limit| rate_limit.get("has_capacity"))
.and_then(admin_provider_quota_pure::coerce_json_bool)
.unwrap_or(false);
has_capacity || !exhausted
}
pub(crate) fn provider_key_status_snapshot_payload(
key: &StoredProviderCatalogKey,
provider_type: &str,
@@ -1963,7 +2271,7 @@ pub(crate) fn build_admin_provider_key_response(
);
payload.insert(
"upstream_metadata".to_string(),
sanitized_admin_upstream_metadata(key.upstream_metadata.as_ref()),
json!(key.upstream_metadata),
);
payload.insert("proxy".to_string(), json!(key.proxy));
payload.insert("fingerprint".to_string(), json!(key.fingerprint));
@@ -2298,6 +2606,274 @@ mod tests {
assert_eq!(auto.get("used_value"), Some(&json!(90.0)));
}
#[test]
fn provider_key_status_snapshot_payload_backfills_windsurf_daily_and_weekly_quota() {
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"plan_name": "Pro",
"daily_remaining_percent": 40.0,
"weekly_remaining_percent": 65.0,
"daily_reset_at": 1_778_100_000u64,
"weekly_reset_at": 1_778_600_000u64,
"prompt_used": 12.0,
"prompt_limit": 100.0,
"prompt_remaining": 88.0,
"flex_used": 3.0,
"flex_limit": 10.0,
"flex_remaining": 7.0,
"allowed_models_count": 82
}
}));
let payload = provider_key_status_snapshot_payload(&key, "windsurf");
let quota = payload
.get("quota")
.and_then(Value::as_object)
.expect("quota snapshot should be object");
let windows = quota
.get("windows")
.and_then(Value::as_array)
.expect("windsurf quota windows should exist");
let daily = windows
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("daily")))
.expect("daily quota window should exist");
let weekly = windows
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("weekly")))
.expect("weekly quota window should exist");
assert_eq!(quota.get("provider_type"), Some(&json!("windsurf")));
assert_eq!(quota.get("code"), Some(&json!("ok")));
assert_eq!(quota.get("plan_type"), Some(&json!("Pro")));
assert_eq!(quota.get("usage_ratio"), Some(&json!(0.6)));
assert_eq!(quota.get("reset_at"), Some(&json!(1_778_100_000u64)));
assert_eq!(daily.get("remaining_ratio"), Some(&json!(0.4)));
assert_eq!(daily.get("used_ratio"), Some(&json!(0.6)));
assert_eq!(daily.get("reset_seconds"), Some(&json!(32_754u64)));
assert_eq!(weekly.get("remaining_ratio"), Some(&json!(0.65)));
assert_eq!(weekly.get("used_ratio"), Some(&json!(0.35)));
assert_eq!(weekly.get("reset_seconds"), Some(&json!(532_754u64)));
assert_eq!(quota.get("allowed_models_count"), Some(&json!(82)));
}
#[test]
fn provider_key_status_snapshot_payload_treats_windsurf_rate_limit_as_cooldown() {
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"daily_remaining_percent": 80.0,
"rate_limit": {
"limited": true,
"retry_after_ms": 60_001u64,
"message": "slow down"
},
"last_error": "slow down"
}
}));
let payload = provider_key_status_snapshot_payload(&key, "windsurf");
let quota = payload
.get("quota")
.and_then(Value::as_object)
.expect("quota snapshot should be object");
let rate_window = quota
.get("windows")
.and_then(Value::as_array)
.and_then(|windows| {
windows
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("rate_limit")))
})
.expect("rate limit window should exist");
assert_eq!(quota.get("code"), Some(&json!("cooldown")));
assert_eq!(quota.get("exhausted"), Some(&json!(false)));
assert_eq!(quota.get("reset_seconds"), Some(&json!(61u64)));
assert_eq!(rate_window.get("is_exhausted"), Some(&json!(false)));
assert_eq!(rate_window.get("reset_seconds"), Some(&json!(61u64)));
}
#[test]
fn provider_key_status_snapshot_payload_keeps_windsurf_capacity_probe_without_retry_after_ok() {
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"daily_remaining_percent": 100.0,
"weekly_remaining_percent": 100.0,
"rate_limit": {
"limited": true,
"has_capacity": false,
"messages_remaining": 0.0,
"max_messages": 100.0
}
}
}));
let payload = provider_key_status_snapshot_payload(&key, "windsurf");
let quota = payload
.get("quota")
.and_then(Value::as_object)
.expect("quota snapshot should be object");
let has_rate_limit_window =
quota
.get("windows")
.and_then(Value::as_array)
.is_some_and(|windows| {
windows
.iter()
.filter_map(Value::as_object)
.any(|window| window.get("code") == Some(&json!("rate_limit")))
});
assert_eq!(quota.get("code"), Some(&json!("ok")));
assert_eq!(quota.get("label"), Some(&Value::Null));
assert_eq!(quota.get("exhausted"), Some(&json!(false)));
assert_eq!(
payload.pointer("/quota/rate_limit/limited"),
Some(&json!(true))
);
assert!(!has_rate_limit_window);
}
#[test]
fn provider_key_status_snapshot_payload_refreshes_stale_windsurf_cooldown_when_probe_has_capacity(
) {
let mut key = sample_catalog_key();
key.status_snapshot = Some(json!({
"quota": {
"version": 2,
"provider_type": "windsurf",
"code": "cooldown",
"label": "冷却中",
"exhausted": false,
"windows": [
{
"code": "daily",
"unit": "percent",
"label": "",
"scope": "account",
"remaining_ratio": 0.99,
"is_exhausted": false
},
{
"code": "rate_limit",
"unit": "count",
"label": "速率",
"scope": "account",
"is_exhausted": false,
"reset_seconds": null
}
],
"rate_limit": {
"limited": true,
"has_capacity": true,
"messages_remaining": -1,
"max_messages": -1
}
}
}));
key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"daily_remaining_percent": 99.0,
"weekly_remaining_percent": 100.0,
"allowed_models_count": 118,
"rate_limit": {
"limited": true,
"has_capacity": true,
"messages_remaining": -1,
"max_messages": -1
}
}
}));
let payload = provider_key_status_snapshot_payload(&key, "windsurf");
let quota = payload
.get("quota")
.and_then(Value::as_object)
.expect("quota snapshot should be object");
let has_rate_limit_window =
quota
.get("windows")
.and_then(Value::as_array)
.is_some_and(|windows| {
windows
.iter()
.filter_map(Value::as_object)
.any(|window| window.get("code") == Some(&json!("rate_limit")))
});
assert_eq!(quota.get("code"), Some(&json!("ok")));
assert_eq!(quota.get("label"), Some(&Value::Null));
assert_eq!(quota.get("allowed_models_count"), Some(&json!(118u64)));
assert!(!has_rate_limit_window);
}
#[test]
fn provider_key_status_snapshot_payload_marks_windsurf_banned_and_quarantined_blocking() {
let mut banned_key = sample_catalog_key();
banned_key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"banned": true,
"reason": "forbidden"
}
}));
let banned_payload = provider_key_status_snapshot_payload(&banned_key, "windsurf");
assert_eq!(
banned_payload.pointer("/quota/code"),
Some(&json!("banned"))
);
assert_eq!(
banned_payload.pointer("/quota/exhausted"),
Some(&json!(true))
);
assert_eq!(
banned_payload.pointer("/account/code"),
Some(&json!("account_banned"))
);
assert_eq!(
banned_payload.pointer("/account/blocked"),
Some(&json!(true))
);
let mut quarantined_key = sample_catalog_key();
quarantined_key.upstream_metadata = Some(json!({
"windsurf": {
"updated_at": 1_778_067_246u64,
"quarantined": true
}
}));
let quarantined_payload =
provider_key_status_snapshot_payload(&quarantined_key, "windsurf");
assert_eq!(
quarantined_payload.pointer("/quota/code"),
Some(&json!("quarantined"))
);
assert_eq!(
quarantined_payload.pointer("/quota/exhausted"),
Some(&json!(true))
);
assert_eq!(
quarantined_payload.pointer("/account/code"),
Some(&json!("account_quarantined"))
);
assert_eq!(
quarantined_payload.pointer("/account/blocked"),
Some(&json!(true))
);
}
#[test]
fn provider_key_status_snapshot_payload_preserves_existing_materialized_quota_snapshot() {
let mut key = sample_catalog_key();

View File

@@ -38,7 +38,8 @@ pub(crate) use self::external_models::OFFICIAL_EXTERNAL_MODEL_PROVIDERS;
pub(crate) use self::normalize::{
deserialize_optional_json_patch, deserialize_optional_string_list_patch, ip_rules_allow,
json_ip_rules_allow, normalize_feature_settings, normalize_ip_rules, normalize_json_array,
normalize_json_object, normalize_string_list, parse_json_ip_rules,
normalize_json_object, normalize_string_list, normalize_user_self_feature_settings_update,
parse_json_ip_rules,
};
pub(crate) use self::payloads::{
InternalGatewayAuthContextRequest, InternalGatewayExecuteRequest,

View File

@@ -54,6 +54,7 @@ pub(crate) fn normalize_feature_settings(value: Option<Value>) -> Result<Option<
Value::Null => Ok(None),
Value::Object(ref mut settings) => {
normalize_chat_pii_redaction_feature_settings(settings)?;
normalize_notification_push_service_feature_settings(settings)?;
if settings.is_empty() {
Ok(None)
} else {
@@ -64,6 +65,41 @@ pub(crate) fn normalize_feature_settings(value: Option<Value>) -> Result<Option<
}
}
pub(crate) fn normalize_user_self_feature_settings_update(
value: Option<Value>,
current: Option<Value>,
) -> Result<Option<Value>, String> {
let mut normalized = normalize_feature_settings(value)?;
let current_notification_push_service = current
.and_then(|value| match value {
Value::Object(mut settings) => settings.remove("notification_push_service"),
_ => None,
})
.and_then(|value| {
let mut wrapper = Map::new();
wrapper.insert("notification_push_service".to_string(), value);
normalize_notification_push_service_feature_settings(&mut wrapper)
.ok()
.and_then(|_| wrapper.remove("notification_push_service"))
});
match (&mut normalized, current_notification_push_service) {
(Some(Value::Object(settings)), Some(value)) => {
settings.insert("notification_push_service".to_string(), value);
}
(Some(Value::Object(settings)), None) => {
settings.remove("notification_push_service");
}
(None, Some(value)) => {
let mut settings = Map::new();
settings.insert("notification_push_service".to_string(), value);
normalized = Some(Value::Object(settings));
}
_ => {}
}
Ok(normalized)
}
pub(crate) fn normalize_ip_rules(
values: Option<Vec<String>>,
) -> Result<Option<Vec<String>>, String> {
@@ -326,9 +362,47 @@ fn normalize_chat_pii_redaction_feature_object(
Ok(())
}
fn normalize_notification_push_service_feature_settings(
settings: &mut Map<String, Value>,
) -> Result<(), String> {
let Some(value) = settings.get_mut("notification_push_service") else {
return Ok(());
};
match value {
Value::Null => {
settings.remove("notification_push_service");
Ok(())
}
Value::Object(feature) => {
normalize_notification_push_service_feature_object(feature)?;
if feature.is_empty() {
settings.remove("notification_push_service");
}
Ok(())
}
_ => Err("notification_push_service 必须是对象".to_string()),
}
}
fn normalize_notification_push_service_feature_object(
feature: &mut Map<String, Value>,
) -> Result<(), String> {
for key in ["enabled"] {
if let Some(value) = feature.get(key) {
if !value.is_boolean() {
return Err(format!("notification_push_service.{key} 必须是布尔值"));
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{ip_rules_allow, json_ip_rules_allow, normalize_ip_rules, parse_json_ip_rules};
use super::{
ip_rules_allow, json_ip_rules_allow, normalize_feature_settings, normalize_ip_rules,
normalize_user_self_feature_settings_update, parse_json_ip_rules,
};
use serde_json::json;
use std::net::{IpAddr, Ipv4Addr};
@@ -358,6 +432,41 @@ mod tests {
);
}
#[test]
fn normalize_feature_settings_accepts_notification_push_service_permission() {
let normalized = normalize_feature_settings(Some(json!({
"notification_push_service": {"enabled": true}
})))
.expect("feature settings should normalize")
.expect("feature settings should remain set");
assert_eq!(
normalized["notification_push_service"]["enabled"],
json!(true)
);
}
#[test]
fn user_self_feature_update_preserves_notification_push_permission() {
let normalized = normalize_user_self_feature_settings_update(
Some(json!({
"chat_pii_redaction": {"enabled": true, "inject_model_instruction": false},
"notification_push_service": {"enabled": false}
})),
Some(json!({
"notification_push_service": {"enabled": true}
})),
)
.expect("feature settings should normalize")
.expect("feature settings should remain set");
assert_eq!(
normalized["notification_push_service"]["enabled"],
json!(true)
);
assert_eq!(normalized["chat_pii_redaction"]["enabled"], json!(true));
}
#[test]
fn ip_rules_allow_applies_allow_rules_and_deny_overrides() {
let rules = vec![

View File

@@ -221,7 +221,6 @@ pub(crate) fn admin_proxy_local_requires_buffered_body(
| (Some("endpoints_manage"), http::Method::POST, Some("create_endpoint"))
| (Some("endpoints_manage"), http::Method::POST, Some("batch_delete_keys"))
| (Some("endpoints_manage"), http::Method::POST, Some("refresh_quota"))
| (Some("endpoints_manage"), http::Method::POST, Some("query_key_balance"))
| (Some("endpoints_manage"), http::Method::PUT, Some("update_key"))
| (Some("endpoints_manage"), http::Method::PUT, Some("update_endpoint"))
| (Some("modules_manage"), http::Method::PUT, Some("set_enabled"))
@@ -255,6 +254,11 @@ pub(crate) fn admin_proxy_local_requires_buffered_body(
| (Some("system_manage"), http::Method::PUT, Some("config_set"))
| (Some("system_manage"), http::Method::PUT, Some("email_template_set"))
| (Some("system_manage"), http::Method::POST, Some("email_template_preview"))
| (
Some("system_manage"),
http::Method::POST,
Some("important_notification_test"),
)
| (
Some("provider_models_manage"),
http::Method::POST,

View File

@@ -0,0 +1,890 @@
use crate::admin_api::AdminAppState;
use crate::bark_push::{read_bark_push_config, send_bark_push, BarkPushConfig};
use crate::email_delivery::{
read_smtp_delivery_config, send_smtp_email, ComposedEmail, SmtpDeliveryConfig,
};
use crate::handlers::shared::{system_config_bool, system_config_string};
use crate::server_chan_push::{
read_server_chan_push_config, send_server_chan_push, ServerChanPushConfig,
};
use crate::{AppState, GatewayError};
use axum::body::Bytes;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tracing::warn;
pub(crate) const IMPORTANT_NOTIFICATION_ENABLED_KEY: &str = "module.important_notification.enabled";
pub(crate) const LEGACY_NOTIFICATION_EMAIL_ENABLED_KEY: &str = "module.notification_email.enabled";
pub(crate) const IMPORTANT_NOTIFICATION_EMAIL_ENABLED_KEY: &str =
"module.important_notification.email_enabled";
pub(crate) const IMPORTANT_NOTIFICATION_EMAIL_RECIPIENTS_KEY: &str =
"module.important_notification.email_recipients";
pub(crate) const IMPORTANT_NOTIFICATION_DEFAULT_CHANNEL_KEY: &str =
"module.important_notification.default_channel";
pub(crate) const IMPORTANT_NOTIFICATION_ITEMS_KEY: &str = "module.important_notification.items";
pub(crate) const PROVIDER_QUOTA_ALERT_ITEM_KEY: &str = "provider_quota_alert";
#[derive(Debug, Clone)]
pub(crate) struct ImportantNotification {
pub(crate) title: String,
pub(crate) markdown_body: String,
pub(crate) text_body: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ImportantNotificationChannelFilter {
All,
Email,
ServerChan,
Bark,
}
#[derive(Debug, Clone)]
struct ImportantNotificationConfig {
module_enabled: bool,
email_enabled: bool,
email_recipients: Vec<String>,
default_channel: ImportantNotificationChannelFilter,
items: Vec<ImportantNotificationItemConfig>,
server_chan: ServerChanPushConfig,
bark: BarkPushConfig,
}
#[derive(Debug, Clone)]
struct ImportantNotificationItemConfig {
key: String,
name: String,
enabled: bool,
channel: Option<ImportantNotificationChannelFilter>,
title_template: Option<String>,
markdown_template: Option<String>,
text_template: Option<String>,
user_email_enabled: bool,
}
#[derive(Debug, Clone, Copy)]
struct NotificationChannelReadiness {
email: bool,
server_chan: bool,
bark: bool,
}
#[derive(Debug, Clone, Serialize)]
pub(crate) struct ImportantNotificationChannelReport {
pub(crate) channel: &'static str,
pub(crate) success: bool,
pub(crate) message: String,
}
#[derive(Debug, Clone, Serialize)]
pub(crate) struct ImportantNotificationDeliveryReport {
pub(crate) success: bool,
pub(crate) channels: Vec<ImportantNotificationChannelReport>,
}
#[derive(Debug, Deserialize)]
struct ImportantNotificationTestRequest {
#[serde(default)]
channel: Option<String>,
#[serde(default)]
item_key: Option<String>,
}
pub(crate) async fn important_notification_module_enabled(
state: &AppState,
) -> Result<bool, GatewayError> {
let canonical = state
.read_system_config_json_value(IMPORTANT_NOTIFICATION_ENABLED_KEY)
.await?;
if canonical.is_some() {
return Ok(system_config_bool(canonical.as_ref(), false));
}
let legacy = state
.read_system_config_json_value(LEGACY_NOTIFICATION_EMAIL_ENABLED_KEY)
.await?;
Ok(system_config_bool(legacy.as_ref(), false))
}
pub(crate) async fn important_notification_configured(
state: &AppState,
) -> Result<bool, GatewayError> {
let config = read_important_notification_config(state).await?;
important_notification_has_configured_channel(state, &config).await
}
pub(crate) async fn important_notification_dispatch_ready_for_item(
state: &AppState,
item_key: &str,
) -> Result<bool, GatewayError> {
let config = read_important_notification_config(state).await?;
if !config.module_enabled {
return Ok(false);
}
if let Some(item) = find_notification_item(&config, item_key) {
if !item.enabled {
return Ok(false);
}
}
let readiness = read_notification_channel_readiness(state, &config).await?;
Ok(channel_filter_has_ready_channel(
notification_item_channel_filter(&config, item_key),
readiness,
))
}
pub(crate) async fn send_important_notification(
state: &AppState,
notification: ImportantNotification,
) -> Result<ImportantNotificationDeliveryReport, GatewayError> {
send_important_notification_with_filter(
state,
notification,
ImportantNotificationChannelFilter::All,
)
.await
}
pub(crate) async fn send_important_notification_for_item(
state: &AppState,
item_key: &str,
notification: ImportantNotification,
variables: &[(&str, String)],
) -> Result<ImportantNotificationDeliveryReport, GatewayError> {
dispatch_important_notification(state, Some(item_key), notification, variables, None, false)
.await
}
pub(crate) async fn send_important_notification_with_filter(
state: &AppState,
notification: ImportantNotification,
channel_filter: ImportantNotificationChannelFilter,
) -> Result<ImportantNotificationDeliveryReport, GatewayError> {
dispatch_important_notification(state, None, notification, &[], Some(channel_filter), false)
.await
}
pub(crate) async fn send_user_important_notification_email(
state: &AppState,
item_key: &str,
user_email: &str,
notification: ImportantNotification,
variables: &[(&str, String)],
) -> Result<ImportantNotificationDeliveryReport, GatewayError> {
let config = read_important_notification_config(state).await?;
if !config.module_enabled {
return Ok(single_report("module", false, "通知服务未启用"));
}
let Some(item) = find_notification_item(&config, item_key) else {
return Ok(single_report("item", false, "通知项未定义"));
};
if !item.enabled {
return Ok(single_report("item", false, "通知项未启用"));
}
if !item.user_email_enabled {
return Ok(single_report("user_email", false, "通知项未启用用户邮件"));
}
let notification = apply_notification_item_template(Some(item), notification, variables);
let smtp_config = match read_smtp_delivery_config(state).await? {
Some(config) => config,
None => return Ok(single_report("user_email", false, "SMTP 配置不完整")),
};
let user_email = user_email.trim();
if user_email.is_empty() {
return Ok(single_report("user_email", false, "用户邮箱为空"));
}
match send_single_email_notification(smtp_config, user_email, &notification).await {
Ok(()) => Ok(single_report("user_email", true, "用户邮件通知已发送")),
Err(err) => {
warn!(error = ?err, user_email = %user_email, "failed to send user notification email");
Ok(single_report(
"user_email",
false,
format!("用户邮件通知发送失败: {err:?}"),
))
}
}
}
async fn important_notification_has_configured_channel(
state: &AppState,
config: &ImportantNotificationConfig,
) -> Result<bool, GatewayError> {
let readiness = read_notification_channel_readiness(state, config).await?;
if channel_filter_has_ready_channel(config.default_channel, readiness) {
return Ok(true);
}
Ok(config.items.iter().any(|item| {
item.enabled
&& channel_filter_has_ready_channel(
item.channel.unwrap_or(config.default_channel),
readiness,
)
}))
}
async fn read_notification_channel_readiness(
state: &AppState,
config: &ImportantNotificationConfig,
) -> Result<NotificationChannelReadiness, GatewayError> {
let smtp_config = read_smtp_delivery_config(state).await?;
Ok(NotificationChannelReadiness {
email: config.email_enabled && !config.email_recipients.is_empty() && smtp_config.is_some(),
server_chan: config.server_chan.enabled && config.server_chan.send_key.is_some(),
bark: config.bark.enabled && config.bark.device_key.is_some(),
})
}
fn channel_filter_has_ready_channel(
filter: ImportantNotificationChannelFilter,
readiness: NotificationChannelReadiness,
) -> bool {
match filter {
ImportantNotificationChannelFilter::All => {
readiness.email || readiness.server_chan || readiness.bark
}
ImportantNotificationChannelFilter::Email => readiness.email,
ImportantNotificationChannelFilter::ServerChan => readiness.server_chan,
ImportantNotificationChannelFilter::Bark => readiness.bark,
}
}
async fn dispatch_important_notification(
state: &AppState,
item_key: Option<&str>,
notification: ImportantNotification,
variables: &[(&str, String)],
channel_override: Option<ImportantNotificationChannelFilter>,
bypass_enable_checks: bool,
) -> Result<ImportantNotificationDeliveryReport, GatewayError> {
let config = read_important_notification_config(state).await?;
if !bypass_enable_checks && !config.module_enabled {
return Ok(single_report("module", false, "通知服务未启用"));
}
let item = item_key.and_then(|key| find_notification_item(&config, key));
if !bypass_enable_checks && item.is_some_and(|item| !item.enabled) {
return Ok(single_report("item", false, "通知项未启用"));
}
let notification = apply_notification_item_template(item, notification, variables);
let channel_filter = channel_override.unwrap_or_else(|| {
item.and_then(|item| item.channel)
.unwrap_or(config.default_channel)
});
let mut reports = Vec::new();
if matches!(
channel_filter,
ImportantNotificationChannelFilter::All | ImportantNotificationChannelFilter::Email
) {
maybe_send_email_notification(
state,
&config,
&notification,
bypass_enable_checks,
&mut reports,
)
.await;
}
if matches!(
channel_filter,
ImportantNotificationChannelFilter::All | ImportantNotificationChannelFilter::ServerChan
) {
maybe_send_server_chan_notification(
state,
&config,
&notification,
bypass_enable_checks,
&mut reports,
)
.await;
}
if matches!(
channel_filter,
ImportantNotificationChannelFilter::All | ImportantNotificationChannelFilter::Bark
) {
maybe_send_bark_notification(
state,
&config,
&notification,
bypass_enable_checks,
&mut reports,
)
.await;
}
if reports.is_empty() {
reports.push(ImportantNotificationChannelReport {
channel: "none",
success: false,
message: "未启用可用的通知通道".to_string(),
});
}
let success = reports.iter().any(|report| report.success);
Ok(ImportantNotificationDeliveryReport {
success,
channels: reports,
})
}
pub(crate) async fn build_important_notification_test_payload(
state: &AdminAppState<'_>,
request_body: Option<&Bytes>,
) -> Result<Value, GatewayError> {
let request = match request_body.filter(|body| !body.is_empty()) {
Some(body) => serde_json::from_slice::<ImportantNotificationTestRequest>(body).unwrap_or(
ImportantNotificationTestRequest {
channel: None,
item_key: None,
},
),
None => ImportantNotificationTestRequest {
channel: None,
item_key: None,
},
};
let filter = request.channel.as_deref().and_then(parse_channel_filter);
let item_key = request
.item_key
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
let variables = vec![
("provider_name", "示例 Provider".to_string()),
("provider_id", "provider-demo".to_string()),
("total_available", "8.0000".to_string()),
("threshold_amount", "10.0000".to_string()),
("user_email", "user@example.com".to_string()),
("balance", "1.0000".to_string()),
];
let report = dispatch_important_notification(
state.app(),
item_key,
ImportantNotification {
title: "Aether 通知服务测试".to_string(),
markdown_body: "这是一条来自 Aether 的通知服务测试。".to_string(),
text_body: "这是一条来自 Aether 的通知服务测试。".to_string(),
},
&variables,
filter,
true,
)
.await?;
Ok(json!({
"success": report.success,
"message": if report.success { "测试通知已发送" } else { "测试通知发送失败" },
"channels": report.channels,
}))
}
async fn read_important_notification_config(
state: &AppState,
) -> Result<ImportantNotificationConfig, GatewayError> {
let module_enabled = important_notification_module_enabled(state).await?;
let email_enabled = state
.read_system_config_json_value(IMPORTANT_NOTIFICATION_EMAIL_ENABLED_KEY)
.await?;
let email_recipients = state
.read_system_config_json_value(IMPORTANT_NOTIFICATION_EMAIL_RECIPIENTS_KEY)
.await?;
let default_channel = state
.read_system_config_json_value(IMPORTANT_NOTIFICATION_DEFAULT_CHANNEL_KEY)
.await?;
let items = state
.read_system_config_json_value(IMPORTANT_NOTIFICATION_ITEMS_KEY)
.await?;
Ok(ImportantNotificationConfig {
module_enabled,
email_enabled: system_config_bool(email_enabled.as_ref(), false),
email_recipients: parse_recipient_list(email_recipients.as_ref()),
default_channel: default_channel
.as_ref()
.and_then(|value| value.as_str())
.and_then(parse_channel_filter)
.unwrap_or(ImportantNotificationChannelFilter::All),
items: parse_notification_items(items.as_ref()),
server_chan: read_server_chan_push_config(state).await?,
bark: read_bark_push_config(state).await?,
})
}
fn parse_channel_filter(raw: &str) -> Option<ImportantNotificationChannelFilter> {
match raw.trim().to_ascii_lowercase().as_str() {
"all" => Some(ImportantNotificationChannelFilter::All),
"email" => Some(ImportantNotificationChannelFilter::Email),
"server_chan" | "serverchan" | "serve_chan" => {
Some(ImportantNotificationChannelFilter::ServerChan)
}
"bark" => Some(ImportantNotificationChannelFilter::Bark),
"global" | "" => None,
_ => None,
}
}
fn parse_notification_items(value: Option<&Value>) -> Vec<ImportantNotificationItemConfig> {
let Some(Value::Array(items)) = value else {
return default_notification_items();
};
items
.iter()
.filter_map(parse_notification_item)
.collect::<Vec<_>>()
}
fn parse_notification_item(value: &Value) -> Option<ImportantNotificationItemConfig> {
let item = value.as_object()?;
let key = item.get("key")?.as_str()?.trim();
if key.is_empty() {
return None;
}
let name = item
.get("name")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(key);
Some(ImportantNotificationItemConfig {
key: key.to_string(),
name: name.to_string(),
enabled: item.get("enabled").and_then(Value::as_bool).unwrap_or(true),
channel: item
.get("channel")
.and_then(Value::as_str)
.and_then(parse_channel_filter),
title_template: optional_non_empty_string(item.get("title_template")),
markdown_template: optional_non_empty_string(item.get("markdown_template")),
text_template: optional_non_empty_string(item.get("text_template")),
user_email_enabled: item
.get("user_email_enabled")
.and_then(Value::as_bool)
.unwrap_or(false),
})
}
fn default_notification_items() -> Vec<ImportantNotificationItemConfig> {
vec![
ImportantNotificationItemConfig {
key: PROVIDER_QUOTA_ALERT_ITEM_KEY.to_string(),
name: "号池额度不足".to_string(),
enabled: true,
channel: None,
title_template: None,
markdown_template: None,
text_template: None,
user_email_enabled: false,
},
ImportantNotificationItemConfig {
key: "provider_pool_abnormal".to_string(),
name: "号池异常".to_string(),
enabled: true,
channel: None,
title_template: Some("号池异常:{provider_name}".to_string()),
markdown_template: Some(
"号池 `{provider_name}` 出现异常,请检查服务状态。".to_string(),
),
text_template: Some("号池 {provider_name} 出现异常,请检查服务状态。".to_string()),
user_email_enabled: false,
},
ImportantNotificationItemConfig {
key: "user_balance_low".to_string(),
name: "用户余额不足".to_string(),
enabled: true,
channel: Some(ImportantNotificationChannelFilter::Email),
title_template: Some("余额不足提醒".to_string()),
markdown_template: Some("你的账户余额已低于提醒阈值,请及时处理。".to_string()),
text_template: Some("你的账户余额已低于提醒阈值,请及时处理。".to_string()),
user_email_enabled: true,
},
]
}
fn optional_non_empty_string(value: Option<&Value>) -> Option<String> {
value
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn find_notification_item<'a>(
config: &'a ImportantNotificationConfig,
item_key: &str,
) -> Option<&'a ImportantNotificationItemConfig> {
let item_key = item_key.trim();
config.items.iter().find(|item| item.key == item_key)
}
fn notification_item_channel_filter(
config: &ImportantNotificationConfig,
item_key: &str,
) -> ImportantNotificationChannelFilter {
find_notification_item(config, item_key)
.and_then(|item| item.channel)
.unwrap_or(config.default_channel)
}
fn apply_notification_item_template(
item: Option<&ImportantNotificationItemConfig>,
notification: ImportantNotification,
variables: &[(&str, String)],
) -> ImportantNotification {
let Some(item) = item else {
return notification;
};
let title = render_template(
item.title_template.as_deref(),
&notification.title,
&notification,
variables,
);
let markdown_body = render_template(
item.markdown_template.as_deref(),
&notification.markdown_body,
&notification,
variables,
);
let text_body = render_template(
item.text_template.as_deref(),
&notification.text_body,
&notification,
variables,
);
ImportantNotification {
title,
markdown_body,
text_body,
}
}
fn render_template(
template: Option<&str>,
fallback: &str,
notification: &ImportantNotification,
variables: &[(&str, String)],
) -> String {
let mut rendered = template
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(fallback)
.to_string();
rendered = rendered
.replace("{title}", &notification.title)
.replace("{body}", &notification.markdown_body)
.replace("{text_body}", &notification.text_body);
for (key, value) in variables {
rendered = rendered.replace(&format!("{{{}}}", key.trim()), value);
}
rendered
}
async fn maybe_send_email_notification(
state: &AppState,
config: &ImportantNotificationConfig,
notification: &ImportantNotification,
bypass_channel_toggle: bool,
reports: &mut Vec<ImportantNotificationChannelReport>,
) {
if !bypass_channel_toggle && !config.email_enabled {
return;
}
if config.email_recipients.is_empty() {
reports.push(ImportantNotificationChannelReport {
channel: "email",
success: false,
message: "未配置邮件收件人".to_string(),
});
return;
}
let smtp_config = match read_smtp_delivery_config(state).await {
Ok(Some(config)) => config,
Ok(None) => {
reports.push(ImportantNotificationChannelReport {
channel: "email",
success: false,
message: "SMTP 配置不完整".to_string(),
});
return;
}
Err(err) => {
reports.push(ImportantNotificationChannelReport {
channel: "email",
success: false,
message: format!("读取 SMTP 配置失败: {err:?}"),
});
return;
}
};
let mut sent = 0usize;
let mut failed = 0usize;
for recipient in &config.email_recipients {
match send_single_email_notification(smtp_config.clone(), recipient, notification).await {
Ok(()) => sent += 1,
Err(err) => {
failed += 1;
warn!(
error = ?err,
recipient = %recipient,
"failed to send important notification email"
);
}
}
}
reports.push(ImportantNotificationChannelReport {
channel: "email",
success: sent > 0,
message: if failed == 0 {
format!("邮件通知已发送给 {sent} 个收件人")
} else {
format!("邮件通知成功 {sent} 个,失败 {failed}")
},
});
}
async fn send_single_email_notification(
smtp_config: SmtpDeliveryConfig,
recipient: &str,
notification: &ImportantNotification,
) -> Result<(), GatewayError> {
send_smtp_email(
smtp_config,
ComposedEmail {
to_email: recipient.to_string(),
subject: notification.title.clone(),
html_body: build_notification_html(notification),
text_body: notification.text_body.clone(),
},
)
.await
}
async fn maybe_send_server_chan_notification(
state: &AppState,
config: &ImportantNotificationConfig,
notification: &ImportantNotification,
bypass_channel_toggle: bool,
reports: &mut Vec<ImportantNotificationChannelReport>,
) {
if !bypass_channel_toggle && !config.server_chan.enabled {
return;
}
if config.server_chan.send_key.is_none() {
reports.push(ImportantNotificationChannelReport {
channel: "server_chan",
success: false,
message: "未配置 Server 酱 SendKey".to_string(),
});
return;
};
match send_server_chan_push(
state,
&config.server_chan,
&notification.title,
&notification.markdown_body,
)
.await
{
Ok(()) => reports.push(ImportantNotificationChannelReport {
channel: "server_chan",
success: true,
message: "Server 酱通知已发送".to_string(),
}),
Err(err) => {
warn!(error = ?err, "failed to send server chan important notification");
reports.push(ImportantNotificationChannelReport {
channel: "server_chan",
success: false,
message: format!("Server 酱通知发送失败: {err:?}"),
});
}
}
}
async fn maybe_send_bark_notification(
state: &AppState,
config: &ImportantNotificationConfig,
notification: &ImportantNotification,
bypass_channel_toggle: bool,
reports: &mut Vec<ImportantNotificationChannelReport>,
) {
if !bypass_channel_toggle && !config.bark.enabled {
return;
}
if config.bark.device_key.is_none() {
reports.push(ImportantNotificationChannelReport {
channel: "bark",
success: false,
message: "未配置 Bark Device Key".to_string(),
});
return;
};
match send_bark_push(
state,
&config.bark,
&notification.title,
&notification.markdown_body,
)
.await
{
Ok(()) => reports.push(ImportantNotificationChannelReport {
channel: "bark",
success: true,
message: "Bark 通知已发送".to_string(),
}),
Err(err) => {
warn!(error = ?err, "failed to send bark important notification");
reports.push(ImportantNotificationChannelReport {
channel: "bark",
success: false,
message: format!("Bark 通知发送失败: {err:?}"),
});
}
}
}
fn single_report(
channel: &'static str,
success: bool,
message: impl Into<String>,
) -> ImportantNotificationDeliveryReport {
ImportantNotificationDeliveryReport {
success,
channels: vec![ImportantNotificationChannelReport {
channel,
success,
message: message.into(),
}],
}
}
fn parse_recipient_list(value: Option<&Value>) -> Vec<String> {
let mut recipients = Vec::new();
match value {
Some(Value::Array(items)) => {
for item in items {
if let Some(raw) = item.as_str() {
push_recipient_parts(&mut recipients, raw);
}
}
}
Some(Value::String(raw)) => push_recipient_parts(&mut recipients, raw),
_ => {}
}
recipients.sort();
recipients.dedup();
recipients
}
fn push_recipient_parts(recipients: &mut Vec<String>, raw: &str) {
for item in raw
.split([',', ';', '\n', '\r'])
.map(str::trim)
.filter(|value| !value.is_empty())
{
recipients.push(item.to_string());
}
}
fn build_notification_html(notification: &ImportantNotification) -> String {
format!(
"<!doctype html><html><body><h2>{}</h2><pre style=\"font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;white-space:pre-wrap;line-height:1.6\">{}</pre></body></html>",
escape_html(&notification.title),
escape_html(&notification.text_body),
)
}
fn escape_html(value: &str) -> String {
value
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&#39;")
}
#[cfg(test)]
mod tests {
use super::{
apply_notification_item_template, parse_channel_filter, parse_notification_items,
parse_recipient_list, ImportantNotification, ImportantNotificationChannelFilter,
};
use serde_json::json;
#[test]
fn parse_recipient_list_accepts_arrays_and_delimiters() {
assert_eq!(
parse_recipient_list(Some(&json!([
"ops@example.com, admin@example.com",
"ops@example.com"
]))),
vec![
"admin@example.com".to_string(),
"ops@example.com".to_string()
]
);
}
#[test]
fn parse_notification_items_reads_channel_and_user_email_flag() {
let items = parse_notification_items(Some(&json!([
{
"key": "user_balance_low",
"name": "用户余额不足",
"enabled": true,
"channel": "email",
"title_template": "余额提醒",
"markdown_template": "{user_email}: {balance}",
"user_email_enabled": true
}
])));
assert_eq!(items.len(), 1);
assert_eq!(items[0].key, "user_balance_low");
assert_eq!(
items[0].channel,
Some(ImportantNotificationChannelFilter::Email)
);
assert!(items[0].user_email_enabled);
}
#[test]
fn parse_channel_filter_accepts_bark() {
assert_eq!(
parse_channel_filter("bark"),
Some(ImportantNotificationChannelFilter::Bark)
);
}
#[test]
fn item_template_renders_fallback_and_variables() {
let items = parse_notification_items(Some(&json!([
{
"key": "provider_quota_alert",
"name": "号池额度不足",
"title_template": "额度提醒:{provider_name}",
"markdown_template": "{body}\n剩余:{total_available}",
"text_template": "{text_body}\n剩余:{total_available}"
}
])));
let rendered = apply_notification_item_template(
Some(&items[0]),
ImportantNotification {
title: "默认标题".to_string(),
markdown_body: "默认正文".to_string(),
text_body: "默认文本".to_string(),
},
&[
("provider_name", "示例 Provider".to_string()),
("total_available", "8.0000".to_string()),
],
);
assert_eq!(rendered.title, "额度提醒:示例 Provider");
assert_eq!(rendered.markdown_body, "默认正文\n剩余8.0000");
assert_eq!(rendered.text_body, "默认文本\n剩余8.0000");
}
}

View File

@@ -30,6 +30,7 @@ mod api;
mod async_task;
mod audit;
mod auth;
mod bark_push;
mod cache;
mod client_session_affinity;
mod clock;
@@ -37,6 +38,7 @@ mod constants;
mod control;
mod data;
mod dispatch;
mod email_delivery;
mod error;
mod execution_runtime;
mod executor;
@@ -46,6 +48,7 @@ mod handlers;
mod headers;
mod hooks;
mod image_capabilities;
mod important_notification;
mod log_ids;
mod maintenance;
pub(crate) mod middleware;
@@ -62,6 +65,7 @@ mod roles;
mod router;
mod routing;
mod scheduler;
mod server_chan_push;
mod state;
mod system_features;
mod task_runtime;

View File

@@ -254,7 +254,7 @@ struct GatewayDataArgs {
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS",
default_value_t = 1
default_value_t = 4
)]
postgres_min_connections: u32,
@@ -497,6 +497,7 @@ impl GatewayUsageArgs {
fn to_config(&self) -> UsageRuntimeConfig {
UsageRuntimeConfig {
enabled: true,
queue_terminal_events: true,
stream_key: self.queue_stream_key.trim().to_string(),
consumer_group: self.queue_group.trim().to_string(),
dlq_stream_key: self.queue_dlq_stream_key.trim().to_string(),
@@ -1670,7 +1671,7 @@ mod tests {
encryption_key: None,
redis_url: None,
redis_key_prefix: None,
postgres_min_connections: 1,
postgres_min_connections: 4,
postgres_max_connections: 20,
postgres_acquire_timeout_ms: 10_000,
postgres_idle_timeout_ms: 30_000,

View File

@@ -7,8 +7,8 @@ pub(crate) use runtime::{
ensure_provider_key_pool_scores_for_keys, inspect_proxy_upgrade_rollout,
list_admin_cleanup_run_records, perform_account_self_check_once,
perform_oauth_token_refresh_once, perform_pool_quota_probe_once, perform_provider_checkin_once,
pool_quota_probe_target_count, preview_manual_usage_cleanup, rebuild_admin_stats_once,
record_completed_cleanup_run, record_proxy_upgrade_traffic_success,
perform_provider_quota_alert_once, pool_quota_probe_target_count, preview_manual_usage_cleanup,
rebuild_admin_stats_once, record_completed_cleanup_run, record_proxy_upgrade_traffic_success,
restore_proxy_upgrade_rollout_skipped_nodes, retry_proxy_upgrade_rollout_node,
run_admin_system_cleanup_once, run_manual_usage_cleanup_once, skip_proxy_upgrade_rollout_node,
spawn_account_self_check_worker, spawn_audit_cleanup_worker, spawn_db_maintenance_worker,
@@ -16,16 +16,17 @@ pub(crate) use runtime::{
spawn_pending_cleanup_worker, spawn_pool_monitor_worker,
spawn_pool_quota_probe_replenish_for_request, spawn_pool_quota_probe_worker,
spawn_pool_score_rebuild_worker, spawn_provider_checkin_worker,
spawn_proxy_node_metrics_cleanup_worker, spawn_proxy_node_stale_cleanup_worker,
spawn_proxy_upgrade_rollout_worker, spawn_request_candidate_cleanup_worker,
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_usage_counter_flush_worker,
spawn_wallet_daily_usage_aggregation_worker, start_admin_request_body_cleanup_task,
start_admin_system_purge_task, start_manual_usage_cleanup_task, start_proxy_upgrade_rollout,
AccountSelfCheckRunSummary, AdminCleanupRunRecord, AdminCleanupTaskKind,
AdminStatsRebuildSummary, AdminSystemCleanupSummary, ManualUsageCleanupError,
ManualUsageCleanupMode, ManualUsageCleanupOptions, OAuthTokenRefreshRunSummary,
PoolQuotaProbeRunSummary, PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary,
spawn_provider_quota_alert_worker, spawn_proxy_node_metrics_cleanup_worker,
spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker,
spawn_request_candidate_cleanup_worker, spawn_stats_aggregation_worker,
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_usage_counter_flush_worker, spawn_wallet_daily_usage_aggregation_worker,
start_admin_request_body_cleanup_task, start_admin_system_purge_task,
start_manual_usage_cleanup_task, start_proxy_upgrade_rollout, AccountSelfCheckRunSummary,
AdminCleanupRunRecord, AdminCleanupTaskKind, AdminStatsRebuildSummary,
AdminSystemCleanupSummary, ManualUsageCleanupError, ManualUsageCleanupMode,
ManualUsageCleanupOptions, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary, ProviderQuotaAlertRunSummary,
ProxyUpgradeRolloutCancelSummary, ProxyUpgradeRolloutConflictClearSummary,
ProxyUpgradeRolloutNodeActionSummary, ProxyUpgradeRolloutProbeConfig,
ProxyUpgradeRolloutSkippedRestoreSummary, ProxyUpgradeRolloutStatus,

View File

@@ -26,6 +26,8 @@ mod pool_quota_probe;
mod pool_score_rebuild;
#[path = "runtime/provider_checkin.rs"]
mod provider_checkin;
#[path = "runtime/provider_quota_alert.rs"]
mod provider_quota_alert;
#[path = "runtime/proxy_node_metrics_cleanup.rs"]
mod proxy_node_metrics_cleanup;
#[path = "runtime/proxy_node_staleness.rs"]
@@ -85,6 +87,9 @@ pub(crate) use pool_score_rebuild::{
PoolScoreRebuildRunSummary, PoolScoreRebuildWorkerConfig,
};
pub(crate) use provider_checkin::{perform_provider_checkin_once, ProviderCheckinRunSummary};
pub(crate) use provider_quota_alert::{
perform_provider_quota_alert_once, ProviderQuotaAlertRunSummary,
};
use proxy_node_metrics_cleanup::*;
use proxy_node_staleness::*;
use proxy_upgrade_rollout::*;
@@ -138,6 +143,8 @@ const USAGE_COUNTER_DELTA_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);
const USAGE_COUNTER_DELTA_CLEANUP_BATCH_SIZE: usize = 5_000;
const USAGE_COUNTER_DELTA_RETENTION_SECS: u64 = 7 * 24 * 60 * 60;
const PROVIDER_CHECKIN_CONCURRENCY: usize = 3;
const PROVIDER_QUOTA_ALERT_CONCURRENCY: usize = 3;
const PROVIDER_QUOTA_ALERT_INTERVAL: Duration = Duration::from_secs(5);
const PROVIDER_CHECKIN_DEFAULT_TIME: &str = "01:05";
const REQUEST_CANDIDATE_CLEANUP_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const STATS_DAILY_AGGREGATION_HOUR: u32 = 0;

View File

@@ -738,8 +738,21 @@ pub(crate) fn spawn_account_self_check_worker(
let mut interval = tokio::time::interval(config.scan_interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if state
.data
.should_defer_maintenance_for_database_pool_pressure(&mut deferred_since)
{
debug!(
event_name = "maintenance_worker_deferred",
log_type = "ops",
worker = "account_self_check",
"gateway account self-check deferred because database pool has no idle reserve"
);
continue;
}
if let Err(err) = perform_account_self_check_once_with_config(&state, config).await {
warn!(
error = ?err,

View File

@@ -393,8 +393,21 @@ pub(crate) fn spawn_pool_score_rebuild_worker(
}
let mut interval = tokio::time::interval(config.interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let mut deferred_since = None;
loop {
interval.tick().await;
if state
.data
.should_defer_maintenance_for_database_pool_pressure(&mut deferred_since)
{
debug!(
event_name = "maintenance_worker_deferred",
log_type = "ops",
worker = "pool_score_rebuild",
"gateway pool score rebuild deferred because database pool has no idle reserve"
);
continue;
}
match perform_pool_score_rebuild_once_with_config(&state, config).await {
Ok(summary) if summary.scores_upserted > 0 => {
info!(

View File

@@ -0,0 +1,486 @@
use std::collections::HashMap;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
};
use futures_util::stream::{self, StreamExt};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tracing::warn;
use crate::admin_api::{
admin_provider_ops_local_action_response, store_admin_provider_ops_balance_cache, AdminAppState,
};
use crate::important_notification::{
important_notification_dispatch_ready_for_item, send_important_notification_for_item,
ImportantNotification, PROVIDER_QUOTA_ALERT_ITEM_KEY,
};
use crate::{AppState, GatewayError};
use super::PROVIDER_QUOTA_ALERT_CONCURRENCY;
const PROVIDER_QUOTA_ALERT_STATE_PREFIX: &str = "provider_ops:quota_alert:";
const PROVIDER_QUOTA_ALERT_DEFAULT_FETCH_INTERVAL_SECS: u64 = 30;
const PROVIDER_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS: u64 = 30;
const PROVIDER_QUOTA_ALERT_REPEAT_COOLDOWN_SECS: u64 = 24 * 60 * 60;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ProviderQuotaAlertRunSummary {
pub(crate) checked: usize,
pub(crate) alerted: usize,
pub(crate) skipped: usize,
pub(crate) failed: usize,
}
#[derive(Debug, Clone)]
struct ProviderQuotaAlertTarget {
provider: StoredProviderCatalogProvider,
config: ProviderQuotaAlertConfig,
}
#[derive(Debug, Clone, Copy)]
struct ProviderQuotaAlertConfig {
enabled: bool,
threshold_amount: f64,
fetch_interval_seconds: u64,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct ProviderQuotaAlertRuntimeState {
#[serde(default)]
last_checked_at: u64,
#[serde(default)]
last_available: Option<f64>,
#[serde(default)]
below_threshold: bool,
#[serde(default)]
last_notified_at: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProviderQuotaAlertStatus {
Checked,
Alerted,
Skipped,
Failed,
}
pub(crate) async fn perform_provider_quota_alert_once(
state: &AppState,
) -> Result<ProviderQuotaAlertRunSummary, GatewayError> {
if !state.has_provider_catalog_data_reader() {
return Ok(ProviderQuotaAlertRunSummary {
checked: 0,
alerted: 0,
skipped: 0,
failed: 0,
});
}
if !important_notification_dispatch_ready_for_item(state, PROVIDER_QUOTA_ALERT_ITEM_KEY).await?
{
return Ok(ProviderQuotaAlertRunSummary {
checked: 0,
alerted: 0,
skipped: 0,
failed: 0,
});
}
let now_unix_secs = now_unix_secs();
let targets = select_provider_quota_alert_targets(state, now_unix_secs).await?;
if targets.is_empty() {
return Ok(ProviderQuotaAlertRunSummary {
checked: 0,
alerted: 0,
skipped: 0,
failed: 0,
});
}
let provider_ids = targets
.iter()
.map(|target| target.provider.id.clone())
.collect::<Vec<_>>();
let mut endpoints_by_provider = HashMap::<String, Vec<StoredProviderCatalogEndpoint>>::new();
for endpoint in state
.list_provider_catalog_endpoints_by_provider_ids(&provider_ids)
.await?
{
endpoints_by_provider
.entry(endpoint.provider_id.clone())
.or_default()
.push(endpoint);
}
let mut results = stream::iter(targets.into_iter().map(|target| {
let state = state.clone();
let provider_id = target.provider.id.clone();
let endpoints = endpoints_by_provider
.get(&provider_id)
.cloned()
.unwrap_or_default();
async move { run_provider_quota_alert_for_provider(&state, target, endpoints).await }
}))
.buffer_unordered(PROVIDER_QUOTA_ALERT_CONCURRENCY);
let mut summary = ProviderQuotaAlertRunSummary {
checked: 0,
alerted: 0,
skipped: 0,
failed: 0,
};
while let Some(status) = results.next().await {
match status {
ProviderQuotaAlertStatus::Checked => summary.checked += 1,
ProviderQuotaAlertStatus::Alerted => {
summary.checked += 1;
summary.alerted += 1;
}
ProviderQuotaAlertStatus::Skipped => summary.skipped += 1,
ProviderQuotaAlertStatus::Failed => summary.failed += 1,
}
}
Ok(summary)
}
async fn select_provider_quota_alert_targets(
state: &AppState,
now_unix_secs: u64,
) -> Result<Vec<ProviderQuotaAlertTarget>, GatewayError> {
let providers = state
.list_provider_catalog_providers(true)
.await?
.into_iter()
.filter_map(|provider| {
let config = provider_quota_alert_config(&provider)?;
(config.enabled).then_some(ProviderQuotaAlertTarget { provider, config })
})
.collect::<Vec<_>>();
let mut due = Vec::new();
for target in providers {
let runtime = read_quota_alert_runtime_state(state, &target.provider.id).await;
let last_checked_at = runtime
.as_ref()
.map(|state| state.last_checked_at)
.unwrap_or(0);
if now_unix_secs.saturating_sub(last_checked_at)
>= target
.config
.fetch_interval_seconds
.max(PROVIDER_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS)
{
due.push(target);
}
}
Ok(due)
}
async fn run_provider_quota_alert_for_provider(
state: &AppState,
target: ProviderQuotaAlertTarget,
endpoints: Vec<StoredProviderCatalogEndpoint>,
) -> ProviderQuotaAlertStatus {
let provider_id = target.provider.id.clone();
let admin_state = AdminAppState::new(state);
let payload = admin_provider_ops_local_action_response(
&admin_state,
&provider_id,
Some(&target.provider),
&endpoints,
"query_balance",
None,
)
.await;
store_admin_provider_ops_balance_cache(&admin_state, &provider_id, &payload).await;
let now_unix_secs = now_unix_secs();
let payload_status_success = payload.get("status").and_then(Value::as_str) == Some("success");
let Some(total_available) = extract_total_available(&payload) else {
warn!(
provider_id = %provider_id,
payload = %payload,
"provider quota alert skipped because balance payload has no total_available"
);
write_checked_runtime_state_without_balance(state, &provider_id, now_unix_secs).await;
return if payload_status_success {
ProviderQuotaAlertStatus::Skipped
} else {
ProviderQuotaAlertStatus::Failed
};
};
let previous = read_quota_alert_runtime_state(state, &provider_id).await;
let should_notify = provider_quota_alert_should_notify(
now_unix_secs,
total_available,
target.config.threshold_amount,
previous.as_ref(),
);
let mut next = ProviderQuotaAlertRuntimeState {
last_checked_at: now_unix_secs,
last_available: Some(total_available),
below_threshold: total_available <= target.config.threshold_amount,
last_notified_at: previous.and_then(|state| state.last_notified_at),
};
if should_notify {
let notification = build_provider_quota_alert_notification(
&target.provider,
total_available,
target.config.threshold_amount,
);
let variables = provider_quota_alert_notification_variables(
&target.provider,
total_available,
target.config.threshold_amount,
);
let report = send_important_notification_for_item(
state,
PROVIDER_QUOTA_ALERT_ITEM_KEY,
notification,
&variables,
)
.await;
let delivered = match &report {
Ok(report) if report.success => true,
Ok(report) => {
warn!(
provider_id = %provider_id,
report = ?report,
"provider quota alert notification did not reach any channel"
);
false
}
Err(err) => {
warn!(
provider_id = %provider_id,
error = ?err,
"provider quota alert notification failed"
);
false
}
};
if delivered {
next.last_notified_at = Some(now_unix_secs);
write_quota_alert_runtime_state(state, &provider_id, &next).await;
return ProviderQuotaAlertStatus::Alerted;
}
write_quota_alert_runtime_state(state, &provider_id, &next).await;
return ProviderQuotaAlertStatus::Failed;
}
write_quota_alert_runtime_state(state, &provider_id, &next).await;
ProviderQuotaAlertStatus::Checked
}
fn provider_quota_alert_config(
provider: &StoredProviderCatalogProvider,
) -> Option<ProviderQuotaAlertConfig> {
let quota_alert = provider
.config
.as_ref()
.and_then(Value::as_object)
.and_then(|config| config.get("provider_ops"))
.and_then(Value::as_object)
.and_then(|provider_ops| provider_ops.get("quota_alert"))
.and_then(Value::as_object)?;
let enabled = quota_alert
.get("enabled")
.and_then(Value::as_bool)
.unwrap_or(false);
let threshold_amount = quota_alert
.get("threshold_amount")
.and_then(value_as_f64)
.filter(|value| value.is_finite() && *value >= 0.0)
.unwrap_or(0.0);
let fetch_interval_seconds = quota_alert
.get("fetch_interval_seconds")
.and_then(Value::as_u64)
.unwrap_or(PROVIDER_QUOTA_ALERT_DEFAULT_FETCH_INTERVAL_SECS)
.max(PROVIDER_QUOTA_ALERT_MIN_FETCH_INTERVAL_SECS);
Some(ProviderQuotaAlertConfig {
enabled,
threshold_amount,
fetch_interval_seconds,
})
}
fn provider_quota_alert_should_notify(
now_unix_secs: u64,
total_available: f64,
threshold_amount: f64,
previous: Option<&ProviderQuotaAlertRuntimeState>,
) -> bool {
if total_available > threshold_amount {
return false;
}
let Some(previous) = previous else {
return true;
};
if !previous.below_threshold {
return true;
}
previous
.last_notified_at
.map(|last| now_unix_secs.saturating_sub(last) >= PROVIDER_QUOTA_ALERT_REPEAT_COOLDOWN_SECS)
.unwrap_or(true)
}
fn extract_total_available(payload: &Value) -> Option<f64> {
if payload.get("status").and_then(Value::as_str) != Some("success") {
return None;
}
payload
.get("data")
.and_then(|data| data.get("total_available"))
.and_then(value_as_f64)
.filter(|value| value.is_finite())
}
fn value_as_f64(value: &Value) -> Option<f64> {
value.as_f64().or_else(|| {
value
.as_str()
.and_then(|raw| raw.trim().parse::<f64>().ok())
})
}
fn build_provider_quota_alert_notification(
provider: &StoredProviderCatalogProvider,
total_available: f64,
threshold_amount: f64,
) -> ImportantNotification {
let title = format!("提供商额度提醒:{}", provider.name);
let body = format!(
"提供商 `{}` 当前剩余额度为 `{:.4}`,已低于或等于提醒阈值 `{:.4}`。\n\nProvider ID: `{}`",
provider.name, total_available, threshold_amount, provider.id
);
let text_body = format!(
"提供商 {} 当前剩余额度为 {:.4},已低于或等于提醒阈值 {:.4}\n\nProvider ID: {}",
provider.name, total_available, threshold_amount, provider.id
);
ImportantNotification {
title,
markdown_body: body,
text_body,
}
}
fn provider_quota_alert_notification_variables(
provider: &StoredProviderCatalogProvider,
total_available: f64,
threshold_amount: f64,
) -> Vec<(&'static str, String)> {
vec![
("provider_name", provider.name.clone()),
("provider_id", provider.id.clone()),
("total_available", format!("{total_available:.4}")),
("threshold_amount", format!("{threshold_amount:.4}")),
]
}
async fn read_quota_alert_runtime_state(
state: &AppState,
provider_id: &str,
) -> Option<ProviderQuotaAlertRuntimeState> {
let key = provider_quota_alert_state_key(provider_id);
state
.runtime_kv_get(&key)
.await
.ok()
.flatten()
.and_then(|raw| serde_json::from_str::<ProviderQuotaAlertRuntimeState>(&raw).ok())
}
async fn write_checked_runtime_state_without_balance(
state: &AppState,
provider_id: &str,
now_unix_secs: u64,
) {
let mut next = read_quota_alert_runtime_state(state, provider_id)
.await
.unwrap_or_default();
next.last_checked_at = now_unix_secs;
write_quota_alert_runtime_state(state, provider_id, &next).await;
}
async fn write_quota_alert_runtime_state(
state: &AppState,
provider_id: &str,
runtime_state: &ProviderQuotaAlertRuntimeState,
) {
let Ok(serialized) = serde_json::to_string(runtime_state) else {
return;
};
if let Err(err) = state
.runtime_state()
.kv_set(
&provider_quota_alert_state_key(provider_id),
serialized,
None,
)
.await
{
warn!(
error = %err,
provider_id,
"failed to write provider quota alert runtime state"
);
}
}
fn provider_quota_alert_state_key(provider_id: &str) -> String {
format!("{PROVIDER_QUOTA_ALERT_STATE_PREFIX}{provider_id}")
}
fn now_unix_secs() -> u64 {
chrono::Utc::now().timestamp().max(0) as u64
}
#[cfg(test)]
mod tests {
use super::{
provider_quota_alert_should_notify, ProviderQuotaAlertRuntimeState,
PROVIDER_QUOTA_ALERT_REPEAT_COOLDOWN_SECS,
};
#[test]
fn quota_alert_notifies_on_first_drop_and_after_cooldown() {
assert!(provider_quota_alert_should_notify(100, 3.0, 5.0, None));
assert!(provider_quota_alert_should_notify(
100,
3.0,
5.0,
Some(&ProviderQuotaAlertRuntimeState {
below_threshold: false,
..ProviderQuotaAlertRuntimeState::default()
})
));
assert!(!provider_quota_alert_should_notify(
100,
3.0,
5.0,
Some(&ProviderQuotaAlertRuntimeState {
below_threshold: true,
last_notified_at: Some(90),
..ProviderQuotaAlertRuntimeState::default()
})
));
assert!(provider_quota_alert_should_notify(
100 + PROVIDER_QUOTA_ALERT_REPEAT_COOLDOWN_SECS,
3.0,
5.0,
Some(&ProviderQuotaAlertRuntimeState {
below_threshold: true,
last_notified_at: Some(100),
..ProviderQuotaAlertRuntimeState::default()
})
));
}
#[test]
fn quota_alert_does_not_notify_above_threshold() {
assert!(!provider_quota_alert_should_notify(100, 6.0, 5.0, None));
}
}

View File

@@ -1,7 +1,8 @@
use std::sync::Arc;
use std::time::Instant;
use chrono::Utc;
use tracing::warn;
use tracing::{debug, warn};
use crate::data::GatewayDataState;
use crate::AppState;
@@ -10,22 +11,22 @@ use super::{
cleanup_processed_usage_counter_deltas_once, duration_until_next_daily_run,
duration_until_next_db_maintenance_run, duration_until_next_stats_aggregation_run,
duration_until_next_stats_hourly_aggregation_run, maintenance_timezone, parse_hhmm_time,
perform_oauth_token_refresh_once, provider_checkin_schedule, run_audit_cleanup_once,
run_db_maintenance_once, run_gemini_file_mapping_cleanup_once, run_pending_cleanup_once,
run_pool_monitor_once, run_provider_checkin_once, run_proxy_node_metrics_cleanup_once,
run_proxy_node_stale_cleanup_once, run_proxy_upgrade_rollout_once,
run_request_candidate_cleanup_once, run_stats_aggregation_once,
perform_oauth_token_refresh_once, perform_provider_quota_alert_once, provider_checkin_schedule,
run_audit_cleanup_once, run_db_maintenance_once, run_gemini_file_mapping_cleanup_once,
run_pending_cleanup_once, run_pool_monitor_once, run_provider_checkin_once,
run_proxy_node_metrics_cleanup_once, run_proxy_node_stale_cleanup_once,
run_proxy_upgrade_rollout_once, run_request_candidate_cleanup_once, run_stats_aggregation_once,
run_stats_hourly_aggregation_once, run_usage_cleanup_once, run_usage_counter_flush_once,
run_wallet_daily_usage_aggregation_once, AUDIT_LOG_CLEANUP_INTERVAL,
GEMINI_FILE_MAPPING_CLEANUP_INTERVAL, OAUTH_TOKEN_REFRESH_INTERVAL, PENDING_CLEANUP_INTERVAL,
POOL_MONITOR_INTERVAL, PROVIDER_CHECKIN_DEFAULT_TIME, PROXY_NODE_METRICS_CLEANUP_HOUR,
PROXY_NODE_METRICS_CLEANUP_MINUTE, PROXY_NODE_STALE_SWEEP_INTERVAL,
PROXY_UPGRADE_ROLLOUT_INTERVAL, REQUEST_CANDIDATE_CLEANUP_INTERVAL, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, USAGE_COUNTER_DELTA_CLEANUP_BATCH_SIZE,
USAGE_COUNTER_DELTA_CLEANUP_INTERVAL, USAGE_COUNTER_DELTA_RETENTION_SECS,
USAGE_COUNTER_FLUSH_BATCH_SIZE, USAGE_COUNTER_FLUSH_CATCH_UP_BURST_LIMIT,
USAGE_COUNTER_FLUSH_INTERVAL, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
POOL_MONITOR_INTERVAL, PROVIDER_CHECKIN_DEFAULT_TIME, PROVIDER_QUOTA_ALERT_INTERVAL,
PROXY_NODE_METRICS_CLEANUP_HOUR, PROXY_NODE_METRICS_CLEANUP_MINUTE,
PROXY_NODE_STALE_SWEEP_INTERVAL, PROXY_UPGRADE_ROLLOUT_INTERVAL,
REQUEST_CANDIDATE_CLEANUP_INTERVAL, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
USAGE_COUNTER_DELTA_CLEANUP_BATCH_SIZE, USAGE_COUNTER_DELTA_CLEANUP_INTERVAL,
USAGE_COUNTER_DELTA_RETENTION_SECS, USAGE_COUNTER_FLUSH_BATCH_SIZE,
USAGE_COUNTER_FLUSH_CATCH_UP_BURST_LIMIT, USAGE_COUNTER_FLUSH_INTERVAL,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
const STATS_DAILY_CATCH_UP_BURST_LIMIT: usize = 14;
@@ -46,6 +47,37 @@ fn log_maintenance_worker_failure(
);
}
fn should_defer_for_database_pressure(
data: &GatewayDataState,
worker: &'static str,
deferred_since: &mut Option<Instant>,
) -> bool {
let Some(summary) = data.database_pool_summary() else {
*deferred_since = None;
return false;
};
if !GatewayDataState::should_defer_maintenance_for_pool_pressure_state(
GatewayDataState::database_pool_summary_under_maintenance_pressure(&summary),
deferred_since,
) {
return false;
}
debug!(
event_name = "maintenance_worker_deferred",
log_type = "ops",
worker,
driver = %summary.driver,
checked_out = summary.checked_out,
pool_size = summary.pool_size,
idle = summary.idle,
max_connections = summary.max_connections,
usage_rate = summary.usage_rate,
"gateway maintenance worker deferred because database pool has no idle reserve"
);
true
}
pub(crate) fn spawn_audit_cleanup_worker(
data: Arc<GatewayDataState>,
) -> Option<tokio::task::JoinHandle<()>> {
@@ -177,8 +209,19 @@ pub(crate) fn spawn_usage_counter_flush_worker(
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut last_delta_cleanup = tokio::time::Instant::now();
let mut usage_counter_flush_deferred_since = None;
let mut usage_counter_delta_cleanup_deferred_since = None;
loop {
if should_defer_for_database_pressure(
&data,
"usage_counter_flush",
&mut usage_counter_flush_deferred_since,
) {
interval.tick().await;
continue;
}
let mut batches = 0_usize;
while batches < USAGE_COUNTER_FLUSH_CATCH_UP_BURST_LIMIT {
match run_usage_counter_flush_once(&data, USAGE_COUNTER_FLUSH_BATCH_SIZE).await {
@@ -197,7 +240,18 @@ pub(crate) fn spawn_usage_counter_flush_worker(
}
if last_delta_cleanup.elapsed() >= USAGE_COUNTER_DELTA_CLEANUP_INTERVAL {
if let Err(err) = cleanup_processed_usage_counter_deltas_once(
if should_defer_for_database_pressure(
&data,
"usage_counter_delta_cleanup",
&mut usage_counter_delta_cleanup_deferred_since,
) {
debug!(
event_name = "maintenance_worker_deferred",
log_type = "ops",
worker = "usage_counter_delta_cleanup",
"gateway maintenance worker deferred cleanup under database pressure"
);
} else if let Err(err) = cleanup_processed_usage_counter_deltas_once(
&data,
USAGE_COUNTER_DELTA_RETENTION_SECS,
USAGE_COUNTER_DELTA_CLEANUP_BATCH_SIZE,
@@ -254,6 +308,34 @@ pub(crate) fn spawn_provider_checkin_worker(
}))
}
pub(crate) fn spawn_provider_quota_alert_worker(
state: AppState,
) -> Option<tokio::task::JoinHandle<()>> {
if !state.has_provider_catalog_data_reader() {
return None;
}
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(PROVIDER_QUOTA_ALERT_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if should_defer_for_database_pressure(
&state.data,
"provider_quota_alert",
&mut deferred_since,
) {
continue;
}
if let Err(err) = perform_provider_quota_alert_once(&state).await {
log_maintenance_worker_failure("provider_quota_alert", "tick", &err);
}
}
}))
}
pub(crate) fn spawn_oauth_token_refresh_worker(
state: AppState,
) -> Option<tokio::task::JoinHandle<()>> {
@@ -268,8 +350,16 @@ pub(crate) fn spawn_oauth_token_refresh_worker(
let mut interval = tokio::time::interval(OAUTH_TOKEN_REFRESH_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if should_defer_for_database_pressure(
&state.data,
"oauth_token_refresh",
&mut deferred_since,
) {
continue;
}
if let Err(err) = perform_oauth_token_refresh_once(&state).await {
log_maintenance_worker_failure("oauth_token_refresh", "tick", &err);
}
@@ -314,8 +404,12 @@ pub(crate) fn spawn_pending_cleanup_worker(
let mut interval = tokio::time::interval(PENDING_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if should_defer_for_database_pressure(&data, "pending_cleanup", &mut deferred_since) {
continue;
}
if let Err(err) = run_pending_cleanup_once(&data).await {
log_maintenance_worker_failure("pending_cleanup", "tick", &err);
}
@@ -337,8 +431,16 @@ pub(crate) fn spawn_proxy_node_stale_cleanup_worker(
let mut interval = tokio::time::interval(PROXY_NODE_STALE_SWEEP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if should_defer_for_database_pressure(
&data,
"proxy_node_stale_cleanup",
&mut deferred_since,
) {
continue;
}
if let Err(err) = run_proxy_node_stale_cleanup_once(&data).await {
log_maintenance_worker_failure("proxy_node_stale_cleanup", "tick", &err);
}
@@ -387,8 +489,16 @@ pub(crate) fn spawn_proxy_upgrade_rollout_worker(
let mut interval = tokio::time::interval(PROXY_UPGRADE_ROLLOUT_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
let mut deferred_since = None;
loop {
interval.tick().await;
if should_defer_for_database_pressure(
&state.data,
"proxy_upgrade_rollout",
&mut deferred_since,
) {
continue;
}
if let Err(err) = run_proxy_upgrade_rollout_once(&state).await {
log_maintenance_worker_failure("proxy_upgrade_rollout", "tick", &err);
}

View File

@@ -426,6 +426,7 @@ mod tests {
keys: Arc<Mutex<Vec<StoredProviderCatalogKey>>>,
transports: Arc<HashMap<(String, String, String), GatewayProviderTransportSnapshot>>,
execution_results: Arc<Mutex<VecDeque<ExecutionResult>>>,
executed_plans: Arc<Mutex<Vec<ExecutionPlan>>>,
cached_models: Arc<Mutex<HashMap<(String, String), Vec<Value>>>>,
}
@@ -443,6 +444,7 @@ mod tests {
keys: Arc::new(Mutex::new(keys)),
transports: Arc::new(transports),
execution_results: Arc::new(Mutex::new(VecDeque::from(execution_results))),
executed_plans: Arc::new(Mutex::new(Vec::new())),
cached_models: Arc::new(Mutex::new(HashMap::new())),
}
}
@@ -482,8 +484,12 @@ mod tests {
async fn execute_model_fetch_execution_plan(
&self,
_plan: &ExecutionPlan,
plan: &ExecutionPlan,
) -> Result<ExecutionResult, String> {
self.executed_plans
.lock()
.expect("executed plans mutex")
.push(plan.clone());
self.execution_results
.lock()
.expect("execution result mutex")
@@ -910,6 +916,134 @@ mod tests {
);
}
#[tokio::test]
async fn model_fetch_fetches_windsurf_model_configs_and_persists_allowed_models() {
let provider = sample_provider("provider-windsurf", "windsurf");
let endpoint = StoredProviderCatalogEndpoint::new(
"endpoint-windsurf-chat".to_string(),
"provider-windsurf".to_string(),
"openai:chat".to_string(),
None,
None,
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://server.codeium.com".to_string(),
None,
None,
None,
None,
None,
None,
None,
)
.expect("endpoint transport should build");
let key = sample_key(
"key-windsurf",
"provider-windsurf",
"api_key",
&["openai:chat"],
);
let mut transport = sample_transport(
"windsurf",
"provider-windsurf",
"endpoint-windsurf-chat",
"key-windsurf",
"openai:chat",
"api_key",
Some(r#"{"provider_type":"windsurf"}"#),
);
transport.endpoint.base_url = "https://server.codeium.com".to_string();
transport.key.decrypted_api_key = "devin-session-token$abc".to_string();
let state = TestState::new(
vec![provider],
vec![endpoint],
vec![key],
HashMap::from([(
(
"provider-windsurf".to_string(),
"endpoint-windsurf-chat".to_string(),
"key-windsurf".to_string(),
),
transport,
)]),
vec![execution_result(json!({
"clientModelConfigs": [
{
"modelUid": "claude-sonnet-4-6",
"label": "Claude Sonnet 4.6",
"provider": "anthropic",
"supportsImages": true,
"creditMultiplier": 4
},
{
"modelUid": "gpt-5.4",
"label": "GPT-5.4",
"provider": "openai"
}
],
"defaultOverrideModelConfig": {
"modelUid": "claude-sonnet-4-6"
}
}))],
);
let summary = perform_model_fetch_once_with_state(&state)
.await
.expect("fetch should succeed");
assert_eq!(summary.succeeded, 1);
let plans = state.executed_plans.lock().expect("executed plans mutex");
assert_eq!(plans.len(), 1);
assert_eq!(
plans[0].url,
"https://server.codeium.com/exa.api_server_pb.ApiServerService/GetCascadeModelConfigs"
);
assert_eq!(plans[0].method, "POST");
assert_eq!(plans[0].provider_api_format, "windsurf:model_configs");
assert_eq!(
plans[0]
.body
.json_body
.as_ref()
.and_then(|body| body.get("metadata"))
.and_then(|metadata| metadata.get("apiKey")),
Some(&json!("devin-session-token$abc"))
);
drop(plans);
let updated = state.key("key-windsurf");
assert_eq!(
updated.allowed_models,
Some(json!(["claude-sonnet-4-6", "gpt-5.4"]))
);
assert_eq!(
updated
.upstream_metadata
.as_ref()
.and_then(|value| value.get("windsurf"))
.and_then(|value| value.get("allowed_models_count")),
Some(&json!(2))
);
assert_eq!(
updated
.upstream_metadata
.as_ref()
.and_then(|value| value.get("windsurf"))
.and_then(|value| value.get("default_model_uid")),
Some(&json!("claude-sonnet-4-6"))
);
let cached = state.cached_models.lock().expect("cache mutex");
let cached_models = cached
.get(&("provider-windsurf".to_string(), "key-windsurf".to_string()))
.expect("cached models should be written");
assert_eq!(
cached_models[0]["api_formats"],
json!(["openai:chat", "openai:responses", "claude:messages"])
);
}
#[tokio::test]
async fn model_fetch_failure_keeps_existing_allowed_models() {
let provider = sample_provider("provider-openai", "openai");

Some files were not shown because too many files have changed in this diff Show More