perf(gateway): scale request hot paths for 20k streams

Shard and singleflight hot-path caches, batch and prioritize candidate and usage lifecycle persistence, and extend database and pressure-test instrumentation for 20k concurrent streams.
2026-10-10 03:09:50 +08:00 · 2026-07-22 02:11:08 +08:00
parent 7756c0913f
commit fc92c4f431
124 changed files with 36325 additions and 3217 deletions
@@ -12,6 +12,7 @@ use aether_contracts::{
    EXECUTION_REQUEST_ACCEPT_INVALID_CERTS_HEADER, EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER,
    TRANSPORT_BACKEND_BROWSER_WREQ, TRANSPORT_HTTP_MODE_AUTO, TRANSPORT_POOL_SCOPE_KEY,
 };
+use aether_data_contracts::repository::provider_catalog::ProviderCatalogKeyRuntimeMetadataUpdate;
 use aether_provider_pool::{
    build_chatgpt_web_pool_quota_request, normalize_chatgpt_web_image_quota_limit,
    ProviderPoolQuotaRequestSpec,
@@ -46,6 +47,7 @@ const CHATGPT_WEB_SEC_CH_UA: &str =
 const CHATGPT_WEB_BROWSER_PROFILE: &str = "chrome143";
 const CHATGPT_WEB_QUOTA_REFRESH_TIMEOUT_MS: u64 = 30_000;
 const CHATGPT_WEB_QUOTA_REFRESH_PROXY_TIMEOUT_MS: u64 = 60_000;
+const RUNTIME_METADATA_CAS_MAX_ATTEMPTS: usize = 16;
 const GPT_IMAGE2_TOKEN_MIN_PIXELS: u64 = 655_360;
 const GPT_IMAGE2_TOKEN_MAX_PIXELS: u64 = 8_294_400;
 const GPT_IMAGE2_TOKEN_MAX_EDGE: u64 = 3_840;
@@ -1057,57 +1059,81 @@ async fn apply_chatgpt_web_image_quota_request_delta(
    if key_id.is_empty() || provider_id.is_empty() {
        return Ok(false);
    }
-    let Some(mut latest_key) = state
-        .read_provider_catalog_keys_by_ids(&[key_id.to_string()])
-        .await
-        .map_err(|err| err.into_message())?
-        .into_iter()
-        .find(|key| key.id == key_id && key.provider_id == provider_id)
-    else {
-        return Ok(false);
-    };
-
-    let mut metadata = latest_key
-        .upstream_metadata
-        .as_ref()
-        .and_then(Value::as_object)
-        .and_then(|metadata| metadata.get("chatgpt_web"))
-        .and_then(Value::as_object)
-        .cloned()
-        .unwrap_or_default();
-
-    let now_unix_secs = current_unix_secs();
    let request_dedup_key = chatgpt_web_image_quota_request_delta_dedup_key(plan);
-    if !apply_chatgpt_web_image_quota_request_delta_to_metadata(
-        &mut metadata,
-        latest_key.status_snapshot.as_ref(),
-        now_unix_secs,
-        request_dedup_key.as_deref(),
-    ) {
-        return Ok(false);
+    for attempt in 0..RUNTIME_METADATA_CAS_MAX_ATTEMPTS {
+        let Some(mut latest_key) = state
+            .read_provider_catalog_keys_by_ids(&[key_id.to_string()])
+            .await
+            .map_err(|err| err.into_message())?
+            .into_iter()
+            .find(|key| key.id == key_id && key.provider_id == provider_id)
+        else {
+            return Ok(false);
+        };
+        let expected_namespace_value = latest_key
+            .upstream_metadata
+            .as_ref()
+            .and_then(Value::as_object)
+            .and_then(|metadata| metadata.get("chatgpt_web"))
+            .cloned();
+        let mut metadata = expected_namespace_value
+            .as_ref()
+            .and_then(Value::as_object)
+            .cloned()
+            .unwrap_or_default();
+        let now_unix_secs = current_unix_secs();
+        if !apply_chatgpt_web_image_quota_request_delta_to_metadata(
+            &mut metadata,
+            latest_key.status_snapshot.as_ref(),
+            now_unix_secs,
+            request_dedup_key.as_deref(),
+        ) {
+            return Ok(false);
+        }
+
+        let namespace_value = Value::Object(metadata);
+        let updated_upstream_metadata = merge_provider_metadata_object(
+            latest_key.upstream_metadata.as_ref(),
+            "chatgpt_web",
+            namespace_value.clone(),
+        );
+        latest_key.upstream_metadata = updated_upstream_metadata;
+        latest_key.status_snapshot = sync_provider_key_quota_status_snapshot(
+            latest_key.status_snapshot.as_ref(),
+            "chatgpt_web",
+            latest_key.upstream_metadata.as_ref(),
+            "image_request_local",
+        );
+        latest_key.status_snapshot = sync_provider_key_oauth_status_snapshot(
+            latest_key.status_snapshot.as_ref(),
+            &latest_key,
+        );
+        latest_key.updated_at_unix_secs = Some(now_unix_secs);
+
+        let persisted = state
+            .update_provider_catalog_key_runtime_metadata(
+                &ProviderCatalogKeyRuntimeMetadataUpdate {
+                    key_id: latest_key.id.clone(),
+                    namespace: "chatgpt_web".to_string(),
+                    expected_upstream_metadata_value: expected_namespace_value,
+                    upstream_metadata_value: namespace_value,
+                    status_snapshot_patch: provider_operational_status_patch(
+                        latest_key.status_snapshot.as_ref(),
+                    ),
+                    updated_at_unix_secs: latest_key.updated_at_unix_secs,
+                },
+            )
+            .await
+            .map_err(|err| err.into_message())?;
+        if persisted {
+            return Ok(true);
+        }
+        if attempt + 1 < RUNTIME_METADATA_CAS_MAX_ATTEMPTS {
+            let backoff_us = 50_u64.saturating_mul((attempt + 1) as u64).min(1_000);
+            tokio::time::sleep(Duration::from_micros(backoff_us)).await;
+        }
    }
-
-    let updated_upstream_metadata = merge_provider_metadata_object(
-        latest_key.upstream_metadata.as_ref(),
-        "chatgpt_web",
-        Value::Object(metadata),
-    );
-    latest_key.upstream_metadata = updated_upstream_metadata;
-    latest_key.status_snapshot = sync_provider_key_quota_status_snapshot(
-        latest_key.status_snapshot.as_ref(),
-        "chatgpt_web",
-        latest_key.upstream_metadata.as_ref(),
-        "image_request_local",
-    );
-    latest_key.status_snapshot =
-        sync_provider_key_oauth_status_snapshot(latest_key.status_snapshot.as_ref(), &latest_key);
-    latest_key.updated_at_unix_secs = Some(now_unix_secs);
-
-    Ok(state
-        .update_provider_catalog_key_runtime_state(&latest_key)
-        .await
-        .map_err(|err| err.into_message())?
-        .is_some())
+    Ok(false)
 }

 fn apply_chatgpt_web_image_quota_request_delta_to_metadata(
@@ -1427,12 +1453,10 @@ async fn refresh_chatgpt_web_image_quota_after_success(

    let body_json = execution_result_json(&result).map_err(|err| err.to_string())?;
    let now_unix_secs = current_unix_secs();
-    let Some(mut metadata) =
-        parse_chatgpt_web_conversation_init_response(&body_json, now_unix_secs)
+    let Some(metadata) = parse_chatgpt_web_conversation_init_response(&body_json, now_unix_secs)
    else {
        return Ok(false);
    };
-
    let Some(latest_key) = state
        .read_provider_catalog_keys_by_ids(&key_ids)
        .await
@@ -1442,9 +1466,17 @@ async fn refresh_chatgpt_web_image_quota_after_success(
    else {
        return Ok(false);
    };
+    let expected_namespace_value = latest_key
+        .upstream_metadata
+        .as_ref()
+        .and_then(Value::as_object)
+        .and_then(|metadata| metadata.get("chatgpt_web"))
+        .cloned();
+    let mut metadata = metadata.clone();
    normalize_chatgpt_web_image_quota_limit(&mut metadata, latest_key.upstream_metadata.as_ref());

    let mut updated_key = latest_key;
+    let namespace_value = metadata.clone();
    let updated_upstream_metadata = merge_provider_metadata_object(
        updated_key.upstream_metadata.as_ref(),
        "chatgpt_web",
@@ -1465,11 +1497,35 @@ async fn refresh_chatgpt_web_image_quota_after_success(
        sync_provider_key_oauth_status_snapshot(updated_key.status_snapshot.as_ref(), &updated_key);
    updated_key.updated_at_unix_secs = Some(now_unix_secs);

-    Ok(state
-        .update_provider_catalog_key_runtime_state(&updated_key)
+    let persisted = state
+        .update_provider_catalog_key_runtime_metadata(&ProviderCatalogKeyRuntimeMetadataUpdate {
+            key_id: updated_key.id.clone(),
+            namespace: "chatgpt_web".to_string(),
+            expected_upstream_metadata_value: expected_namespace_value,
+            upstream_metadata_value: namespace_value,
+            status_snapshot_patch: provider_operational_status_patch(
+                updated_key.status_snapshot.as_ref(),
+            ),
+            updated_at_unix_secs: updated_key.updated_at_unix_secs,
+        })
        .await
-        .map_err(|err| err.into_message())?
-        .is_some())
+        .map_err(|err| err.into_message())?;
+    if persisted {
+        return state
+            .update_provider_catalog_key_oauth_runtime_state(
+                &updated_key.id,
+                updated_key.oauth_invalid_at_unix_secs,
+                updated_key.oauth_invalid_reason.as_deref(),
+                None,
+                updated_key.updated_at_unix_secs,
+            )
+            .await
+            .map_err(|err| err.into_message());
+    }
+    // The conversation/init response is an authoritative snapshot.  A
+    // conflict means a newer local delta won; do not overwrite it with
+    // this stale response.  The next refresh will observe the new value.
+    Ok(false)
 }

 fn build_chatgpt_web_image_quota_refresh_plan(
@@ -1557,6 +1613,18 @@ fn merge_provider_metadata_object(
    Some(Value::Object(merged))
 }

+fn provider_operational_status_patch(status_snapshot: Option<&Value>) -> Value {
+    let mut patch = Map::new();
+    if let Some(snapshot) = status_snapshot.and_then(Value::as_object) {
+        for field in ["quota", "oauth"] {
+            if let Some(value) = snapshot.get(field) {
+                patch.insert(field.to_string(), value.clone());
+            }
+        }
+    }
+    Value::Object(patch)
+}
+
 fn chatgpt_web_image_quota_snapshot_window(
    status_snapshot: Option<&Value>,
 ) -> Option<&Map<String, Value>> {