fix(provider): 修复 Windsurf 原生工具桥接

2026-10-04 08:27:46 +08:00 · 2026-05-21 01:02:02 +08:00
parent 82637ad882
commit 129c7c90c0
30 changed files with 9090 additions and 194 deletions
@@ -5,8 +5,11 @@ use super::local_monitoring_response;
 use crate::data::GatewayDataState;
 use crate::AppState;
 use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
-use aether_data_contracts::repository::candidates::RequestCandidateStatus;
+use aether_data_contracts::repository::{
+    candidates::RequestCandidateStatus, usage::UsageBodyCaptureState,
+};
 use axum::body::to_bytes;
+use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
 use serde_json::json;
 use std::sync::Arc;

@@ -530,6 +533,89 @@ async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_respon
    assert!(extra.get("provider_response").is_none());
 }

+#[tokio::test]
+async fn admin_monitoring_trace_request_decodes_connect_json_response_body_refs() {
+    let mut candidate = sample_candidate(
+        "cand-used",
+        "request-connect",
+        0,
+        RequestCandidateStatus::Failed,
+        Some(101),
+        Some(33),
+        Some(429),
+    );
+    candidate.extra_data = Some(json!({"cache_1h": true}));
+
+    let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
+    let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
+        vec![sample_provider()],
+        vec![sample_endpoint()],
+        vec![sample_key()],
+    ));
+    let mut usage = sample_usage(
+        "request-connect",
+        "provider-1",
+        "Windsurf",
+        0,
+        0.0,
+        "failed",
+        Some(429),
+        100,
+    );
+    usage.candidate_id = Some("cand-used".to_string());
+    usage.response_headers = Some(json!({
+        "content-type": "application/connect+json"
+    }));
+    let mut framed = Vec::new();
+    framed.push(2);
+    let payload = br#"{"error":{"code":"resource_exhausted","message":"quota exhausted"}}"#;
+    framed.extend_from_slice(&(payload.len() as u32).to_be_bytes());
+    framed.extend_from_slice(payload);
+    usage.response_body = Some(json!(BASE64_STANDARD.encode(framed)));
+    usage.response_body_ref = Some("usage://request/request-connect/response_body".to_string());
+    usage.response_body_state = Some(UsageBodyCaptureState::Inline);
+    let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
+    let data_state =
+        crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
+            request_candidates,
+            usage_repository,
+        )
+        .with_provider_catalog_reader(provider_catalog);
+    let state = AppState::new()
+        .expect("state should build")
+        .with_data_state_for_tests(data_state);
+    let context = request_context(
+        http::Method::GET,
+        "/api/admin/monitoring/trace/request-connect",
+    );
+
+    let response = local_monitoring_response(&state, &context)
+        .await
+        .expect("handler should not error")
+        .expect("route should be handled locally");
+
+    assert_eq!(response.status(), http::StatusCode::OK);
+    let body = to_bytes(response.into_body(), usize::MAX)
+        .await
+        .expect("body should read");
+    let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
+    let upstream_response = &payload["candidates"][0]["extra_data"]["upstream_response"];
+    assert_eq!(upstream_response["status_code"], json!(429));
+    assert_eq!(
+        upstream_response["body"]["error"]["code"],
+        json!("resource_exhausted")
+    );
+    assert_eq!(
+        upstream_response["body"]["error"]["message"],
+        json!("quota exhausted")
+    );
+    assert_eq!(
+        upstream_response["body_ref"],
+        json!("usage://request/request-connect/response_body")
+    );
+    assert_eq!(upstream_response["body_state"], json!("inline"));
+}
+
 #[tokio::test]
 async fn admin_monitoring_trace_request_exposes_structured_ranking_metadata() {
    let mut candidate = sample_candidate(
@@ -3158,17 +3158,22 @@ async fn provider_query_execute_windsurf_test_candidate(
    }

    let incoming_request_headers = provider_query_extract_request_headers(payload);
-    let mut request_body = original_request_body.clone();
-    if let Some(object) = request_body.as_object_mut() {
-        object.insert("stream".to_string(), Value::Bool(false));
-    }
+    let request_body = original_request_body.clone();
    let request_model =
        provider_query_request_body_model(&request_body, &candidate.effective_model);
-    let upstream_is_stream = provider_query_resolve_standard_test_upstream_is_stream(
-        transport.endpoint.config.as_ref(),
+    let client_is_stream = request_body
+        .get("stream")
+        .and_then(Value::as_bool)
+        .unwrap_or(false);
+    let hard_requires_streaming = crate::ai_serving::force_upstream_streaming_for_provider(
        transport.provider.provider_type.as_str(),
        candidate.endpoint.api_format.as_str(),
    );
+    let upstream_is_stream = crate::ai_serving::resolve_upstream_is_stream_from_endpoint_config(
+        transport.endpoint.config.as_ref(),
+        client_is_stream,
+        hard_requires_streaming,
+    );
    let Some((auth_header, auth_value)) =
        crate::provider_transport::windsurf::resolve_windsurf_cascade_auth(&transport).or_else(
            || crate::provider_transport::auth::resolve_local_openai_bearer_auth(&transport),
@@ -42,9 +42,9 @@ pub(super) fn provider_query_test_attempt_payload(
        "status_code": execution.status_code,
        "latency_ms": execution.latency_ms,
        "request_url": execution.request_url,
-        "request_headers": provider_query_redact_diagnostic_headers(&execution.request_headers),
-        "request_body": execution.request_body,
-        "response_headers": provider_query_redact_diagnostic_headers(&execution.response_headers),
+        "request_headers": redacted_provider_query_headers(&execution.request_headers),
+        "request_body": redacted_provider_query_value(&execution.request_body),
+        "response_headers": redacted_provider_query_headers(&execution.response_headers),
        "response_body": execution.response_body,
    })
 }
@@ -172,34 +172,84 @@ fn provider_query_endpoint_route_payload(
    })
 }

-fn provider_query_redact_diagnostic_headers(
-    headers: &BTreeMap<String, String>,
-) -> BTreeMap<String, String> {
+fn redacted_provider_query_headers(headers: &BTreeMap<String, String>) -> BTreeMap<String, String> {
    headers
        .iter()
-        .map(|(name, value)| {
-            if provider_query_header_is_sensitive(name) {
-                (name.clone(), "<redacted>".to_string())
+        .map(|(key, value)| {
+            if provider_query_field_is_sensitive(key) {
+                (key.clone(), "[REDACTED]".to_string())
            } else {
-                (name.clone(), value.clone())
+                (key.clone(), value.clone())
            }
        })
        .collect()
 }

-fn provider_query_header_is_sensitive(name: &str) -> bool {
+fn redacted_provider_query_value(value: &Value) -> Value {
+    match value {
+        Value::Object(object) => Value::Object(
+            object
+                .iter()
+                .map(|(key, value)| {
+                    if provider_query_field_is_sensitive(key) {
+                        (key.clone(), Value::String("[REDACTED]".to_string()))
+                    } else {
+                        (key.clone(), redacted_provider_query_value(value))
+                    }
+                })
+                .collect(),
+        ),
+        Value::Array(items) => Value::Array(
+            items
+                .iter()
+                .map(redacted_provider_query_value)
+                .collect::<Vec<_>>(),
+        ),
+        other => other.clone(),
+    }
+}
+
+fn provider_query_field_is_sensitive(key: &str) -> bool {
+    let key = key.trim().to_ascii_lowercase();
+    let normalized = key
+        .chars()
+        .filter(|ch| ch.is_ascii_alphanumeric())
+        .collect::<String>();
+    if matches!(
+        normalized.as_str(),
+        "maxtokens"
+            | "maxoutputtokens"
+            | "inputtokens"
+            | "outputtokens"
+            | "prompttokens"
+            | "completiontokens"
+            | "totaltokens"
+    ) {
+        return false;
+    }
    matches!(
-        name.trim().to_ascii_lowercase().as_str(),
+        key.as_str(),
        "authorization"
            | "proxy-authorization"
            | "cookie"
            | "set-cookie"
-            | "x-api-key"
+            | "api_key"
+            | "apikey"
            | "api-key"
+            | "x-api-key"
            | "x-goog-api-key"
            | "anthropic-api-key"
            | "openai-api-key"
-    )
+            | "x-codeium-csrf-token"
+            | "access_token"
+            | "refresh_token"
+            | "id_token"
+            | "password"
+            | "secret"
+    ) || normalized.ends_with("token")
+        || normalized.contains("secret")
+        || normalized.contains("apikey")
+        || normalized.contains("authorization")
 }

 pub(super) fn provider_query_candidate_summary_payload(
@@ -309,34 +359,93 @@ pub(super) fn provider_query_candidate_summary_payload(

 #[cfg(test)]
 mod tests {
-    use super::*;
+    use super::{redacted_provider_query_headers, redacted_provider_query_value};
+    use serde_json::json;
+    use std::collections::BTreeMap;

    #[test]
-    fn provider_query_diagnostic_headers_redact_credentials() {
+    fn redacts_sensitive_provider_query_headers() {
        let headers = BTreeMap::from([
            ("cookie".to_string(), "sso=secret".to_string()),
-            ("authorization".to_string(), "Bearer secret".to_string()),
+            (
+                "authorization".to_string(),
+                "Bearer secret-token".to_string(),
+            ),
            ("x-goog-api-key".to_string(), "secret".to_string()),
            ("content-type".to_string(), "application/json".to_string()),
+            (
+                "x-codeium-csrf-token".to_string(),
+                "csrf-secret".to_string(),
+            ),
        ]);

-        let redacted = provider_query_redact_diagnostic_headers(&headers);
+        let redacted = redacted_provider_query_headers(&headers);

        assert_eq!(
            redacted.get("cookie").map(String::as_str),
-            Some("<redacted>")
+            Some("[REDACTED]")
        );
        assert_eq!(
            redacted.get("authorization").map(String::as_str),
-            Some("<redacted>")
+            Some("[REDACTED]")
        );
        assert_eq!(
            redacted.get("x-goog-api-key").map(String::as_str),
-            Some("<redacted>")
+            Some("[REDACTED]")
+        );
+        assert_eq!(
+            redacted.get("x-codeium-csrf-token").map(String::as_str),
+            Some("[REDACTED]")
        );
        assert_eq!(
            redacted.get("content-type").map(String::as_str),
            Some("application/json")
        );
    }
+
+    #[test]
+    fn redacts_sensitive_provider_query_request_body_fields() {
+        let body = json!({
+            "metadata": {
+                "apiKey": "devin-session-token$secret",
+                "ideName": "windsurf"
+            },
+            "messages": [{"role": "user", "content": "hello"}],
+            "stream": true
+        });
+
+        let redacted = redacted_provider_query_value(&body);
+
+        assert_eq!(
+            redacted.pointer("/metadata/apiKey"),
+            Some(&json!("[REDACTED]"))
+        );
+        assert_eq!(
+            redacted.pointer("/metadata/ideName"),
+            Some(&json!("windsurf"))
+        );
+        assert_eq!(redacted.pointer("/stream"), Some(&json!(true)));
+    }
+
+    #[test]
+    fn keeps_non_secret_token_count_fields_visible() {
+        let body = json!({
+            "maxTokens": 64,
+            "usage": {
+                "inputTokens": 10,
+                "outputTokens": 2,
+                "accessToken": "secret"
+            }
+        });
+
+        let redacted = redacted_provider_query_value(&body);
+
+        assert_eq!(redacted.pointer("/maxTokens"), Some(&json!(64)));
+        assert_eq!(redacted.pointer("/usage/inputTokens"), Some(&json!(10)));
+        assert_eq!(redacted.pointer("/usage/outputTokens"), Some(&json!(2)));
+        assert_eq!(
+            redacted.pointer("/usage/accessToken"),
+            Some(&json!("[REDACTED]"))
+        );
+    }
 }
@@ -1268,17 +1268,11 @@ fn build_windsurf_quota_status_snapshot(
            let retry_after_ms = rate_limit_object
                .get("retry_after_ms")
                .or_else(|| rate_limit_object.get("retryAfterMs"))
-                .and_then(admin_provider_quota_pure::coerce_json_u64);
-            let limited = rate_limit_object
-                .get("limited")
-                .or_else(|| rate_limit_object.get("is_limited"))
-                .or_else(|| rate_limit_object.get("isLimited"))
-                .and_then(admin_provider_quota_pure::coerce_json_bool)
-                == Some(true);
-            if limited || retry_after_ms.is_some() {
+                .and_then(admin_provider_quota_pure::coerce_json_u64)
+                .filter(|value| *value > 0);
+            if let Some(retry_after_ms) = retry_after_ms {
                rate_limit_cooling = true;
-                rate_limit_reset_seconds =
-                    retry_after_ms.map(|value| value.saturating_add(999) / 1000);
+                rate_limit_reset_seconds = Some(retry_after_ms.saturating_add(999) / 1000);
                rate_limit_reason = rate_limit_object
                    .get("message")
                    .and_then(Value::as_str)
@@ -1674,6 +1668,12 @@ fn quota_snapshot_has_materialized_data(
        return false;
    }

+    if normalized_provider_type == "windsurf"
+        && windsurf_quota_snapshot_has_stale_cooldown(quota_snapshot)
+    {
+        return false;
+    }
+
    if quota_snapshot
        .get("windows")
        .and_then(Value::as_array)
@@ -1699,6 +1699,61 @@ fn quota_snapshot_has_materialized_data(
        })
 }

+fn windsurf_quota_snapshot_has_stale_cooldown(quota_snapshot: &Map<String, Value>) -> bool {
+    let code = quota_snapshot
+        .get("code")
+        .and_then(Value::as_str)
+        .map(str::trim)
+        .unwrap_or_default();
+    if !code.eq_ignore_ascii_case("cooldown") {
+        return false;
+    }
+
+    let rate_limit = quota_snapshot.get("rate_limit").and_then(Value::as_object);
+    let retry_after_ms = rate_limit
+        .and_then(|rate_limit| {
+            rate_limit
+                .get("retry_after_ms")
+                .or_else(|| rate_limit.get("retryAfterMs"))
+                .and_then(admin_provider_quota_pure::coerce_json_u64)
+        })
+        .unwrap_or(0);
+    if retry_after_ms > 0 {
+        return false;
+    }
+
+    let has_positive_rate_limit_reset = quota_snapshot
+        .get("windows")
+        .and_then(Value::as_array)
+        .is_some_and(|windows| {
+            windows.iter().filter_map(Value::as_object).any(|window| {
+                window
+                    .get("code")
+                    .and_then(Value::as_str)
+                    .is_some_and(|code| code.eq_ignore_ascii_case("rate_limit"))
+                    && window
+                        .get("reset_seconds")
+                        .or_else(|| window.get("reset_at"))
+                        .and_then(admin_provider_quota_pure::coerce_json_u64)
+                        .is_some_and(|value| value > 0)
+            })
+        });
+    if has_positive_rate_limit_reset {
+        return false;
+    }
+
+    let exhausted = quota_snapshot
+        .get("exhausted")
+        .and_then(admin_provider_quota_pure::coerce_json_bool)
+        .unwrap_or(false);
+    let has_capacity = rate_limit
+        .and_then(|rate_limit| rate_limit.get("has_capacity"))
+        .and_then(admin_provider_quota_pure::coerce_json_bool)
+        .unwrap_or(false);
+
+    has_capacity || !exhausted
+}
+
 pub(crate) fn provider_key_status_snapshot_payload(
    key: &StoredProviderCatalogKey,
    provider_type: &str,
@@ -2671,6 +2726,123 @@ mod tests {
        assert_eq!(rate_window.get("reset_seconds"), Some(&json!(61u64)));
    }

+    #[test]
+    fn provider_key_status_snapshot_payload_keeps_windsurf_capacity_probe_without_retry_after_ok() {
+        let mut key = sample_catalog_key();
+        key.upstream_metadata = Some(json!({
+            "windsurf": {
+                "updated_at": 1_778_067_246u64,
+                "daily_remaining_percent": 100.0,
+                "weekly_remaining_percent": 100.0,
+                "rate_limit": {
+                    "limited": true,
+                    "has_capacity": false,
+                    "messages_remaining": 0.0,
+                    "max_messages": 100.0
+                }
+            }
+        }));
+
+        let payload = provider_key_status_snapshot_payload(&key, "windsurf");
+        let quota = payload
+            .get("quota")
+            .and_then(Value::as_object)
+            .expect("quota snapshot should be object");
+        let has_rate_limit_window =
+            quota
+                .get("windows")
+                .and_then(Value::as_array)
+                .is_some_and(|windows| {
+                    windows
+                        .iter()
+                        .filter_map(Value::as_object)
+                        .any(|window| window.get("code") == Some(&json!("rate_limit")))
+                });
+
+        assert_eq!(quota.get("code"), Some(&json!("ok")));
+        assert_eq!(quota.get("label"), Some(&Value::Null));
+        assert_eq!(quota.get("exhausted"), Some(&json!(false)));
+        assert_eq!(
+            payload.pointer("/quota/rate_limit/limited"),
+            Some(&json!(true))
+        );
+        assert!(!has_rate_limit_window);
+    }
+
+    #[test]
+    fn provider_key_status_snapshot_payload_refreshes_stale_windsurf_cooldown_when_probe_has_capacity(
+    ) {
+        let mut key = sample_catalog_key();
+        key.status_snapshot = Some(json!({
+            "quota": {
+                "version": 2,
+                "provider_type": "windsurf",
+                "code": "cooldown",
+                "label": "冷却中",
+                "exhausted": false,
+                "windows": [
+                    {
+                        "code": "daily",
+                        "unit": "percent",
+                        "label": "日",
+                        "scope": "account",
+                        "remaining_ratio": 0.99,
+                        "is_exhausted": false
+                    },
+                    {
+                        "code": "rate_limit",
+                        "unit": "count",
+                        "label": "速率",
+                        "scope": "account",
+                        "is_exhausted": false,
+                        "reset_seconds": null
+                    }
+                ],
+                "rate_limit": {
+                    "limited": true,
+                    "has_capacity": true,
+                    "messages_remaining": -1,
+                    "max_messages": -1
+                }
+            }
+        }));
+        key.upstream_metadata = Some(json!({
+            "windsurf": {
+                "updated_at": 1_778_067_246u64,
+                "daily_remaining_percent": 99.0,
+                "weekly_remaining_percent": 100.0,
+                "allowed_models_count": 118,
+                "rate_limit": {
+                    "limited": true,
+                    "has_capacity": true,
+                    "messages_remaining": -1,
+                    "max_messages": -1
+                }
+            }
+        }));
+
+        let payload = provider_key_status_snapshot_payload(&key, "windsurf");
+        let quota = payload
+            .get("quota")
+            .and_then(Value::as_object)
+            .expect("quota snapshot should be object");
+        let has_rate_limit_window =
+            quota
+                .get("windows")
+                .and_then(Value::as_array)
+                .is_some_and(|windows| {
+                    windows
+                        .iter()
+                        .filter_map(Value::as_object)
+                        .any(|window| window.get("code") == Some(&json!("rate_limit")))
+                });
+
+        assert_eq!(quota.get("code"), Some(&json!("ok")));
+        assert_eq!(quota.get("label"), Some(&Value::Null));
+        assert_eq!(quota.get("allowed_models_count"), Some(&json!(118u64)));
+        assert!(!has_rate_limit_window);
+    }
+
    #[test]
    fn provider_key_status_snapshot_payload_marks_windsurf_banned_and_quarantined_blocking() {
        let mut banned_key = sample_catalog_key();