Make pool probing request-driven

2026-09-02 01:10:23 +08:00 · 2026-05-16 00:18:47 +08:00
parent 43d891bee1
commit d53546d56f
10 changed files with 78 additions and 69 deletions
--- a/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs
+++ b/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs
@@ -407,7 +407,6 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
            overload_cooldown_seconds: 30,
            health_policy_enabled: true,
            probing_enabled: false,
-            probing_interval_minutes: 10,
            probing_target_percent: None,
            probing_target_count: None,
            probe_concurrency: 4,
@@ -475,12 +474,6 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
            .get("probing_enabled")
            .and_then(Value::as_bool)
            .unwrap_or(false),
-        probing_interval_minutes: pool_advanced
-            .get("probing_interval_minutes")
-            .and_then(json_u64)
-            .filter(|value| *value > 0)
-            .map(|value| value.min(1440))
-            .unwrap_or(10),
        probing_target_percent: parse_pool_probe_target_percent(pool_advanced),
        probing_target_count: parse_pool_probe_target_count(pool_advanced),
        probe_concurrency: pool_advanced
@@ -592,7 +585,6 @@ mod tests {
                "overload_cooldown_seconds": 45,
                "health_policy_enabled": false,
                "probing_enabled": true,
-                "probing_interval_minutes": 20,
                "probing_target_percent": 25,
                "probing_target_count": 3,
                "probe_concurrency": 6,
@@ -634,7 +626,6 @@ mod tests {
        assert_eq!(config.overload_cooldown_seconds, 45);
        assert!(!config.health_policy_enabled);
        assert!(config.probing_enabled);
-        assert_eq!(config.probing_interval_minutes, 20);
        assert_eq!(config.probing_target_percent, Some(25.0));
        assert_eq!(config.probing_target_count, Some(3));
        assert_eq!(config.probe_concurrency, 6);
@@ -656,7 +647,7 @@ mod tests {
    }

    #[test]
-    fn clamps_pool_quota_probe_interval_to_python_range() {
+    fn ignores_legacy_pool_quota_probe_interval() {
        let provider = sample_provider(json!({
            "pool_advanced": {
                "probing_enabled": true,
@@ -664,7 +655,7 @@ mod tests {
            }
        }));
        let config = admin_provider_pool_config(&provider).expect("pool config should exist");
-        assert_eq!(config.probing_interval_minutes, 1440);
+        assert!(config.probing_enabled);

        let provider = sample_provider(json!({
            "pool_advanced": {
@@ -673,7 +664,7 @@ mod tests {
            }
        }));
        let config = admin_provider_pool_config(&provider).expect("pool config should exist");
-        assert_eq!(config.probing_interval_minutes, 10);
+        assert!(config.probing_enabled);
    }

    #[test]
--- a/apps/aether-gateway/src/handlers/admin/provider/pool/runtime/writes.rs
+++ b/apps/aether-gateway/src/handlers/admin/provider/pool/runtime/writes.rs
@@ -614,7 +614,6 @@ mod tests {
            overload_cooldown_seconds: 30,
            health_policy_enabled: true,
            probing_enabled: false,
-            probing_interval_minutes: 10,
            probing_target_percent: None,
            probing_target_count: None,
            probe_concurrency: 4,
--- a/apps/aether-gateway/src/handlers/admin/provider/shared/support.rs
+++ b/apps/aether-gateway/src/handlers/admin/provider/shared/support.rs
@@ -45,7 +45,6 @@ pub(crate) struct AdminProviderPoolConfig {
    pub(crate) overload_cooldown_seconds: u64,
    pub(crate) health_policy_enabled: bool,
    pub(crate) probing_enabled: bool,
-    pub(crate) probing_interval_minutes: u64,
    pub(crate) probing_target_percent: Option<f64>,
    pub(crate) probing_target_count: Option<u64>,
    pub(crate) probe_concurrency: u64,
--- a/apps/aether-gateway/src/maintenance/runtime/pool_quota_probe.rs
+++ b/apps/aether-gateway/src/maintenance/runtime/pool_quota_probe.rs
@@ -29,7 +29,7 @@ use crate::handlers::shared::provider_pool::{
 };
 use crate::provider_pool_demand::{
    provider_pool_burst_pending_key, read_provider_pool_demand_snapshot,
-    sample_provider_pool_demand,
+    sample_provider_pool_demand, ProviderPoolDemandSnapshot,
 };

 use super::pool_score_rebuild::ensure_provider_key_pool_scores_for_keys;
@@ -44,6 +44,9 @@ const POOL_QUOTA_PROBE_BURST_TRIGGER_LOCK_TTL_MS: u64 = 30_000;
 const POOL_QUOTA_PROBE_BURST_PENDING_PREFIX: &str = "ap:quota_probe:burst_pending";
 const POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS: u64 = 30;
 const POOL_QUOTA_PROBE_BURST_RETRY_GUARD_SECONDS: u64 = 15;
+const POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS: u64 = 30;
+const POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS: u64 = 10 * 60;
+const POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE: u64 = 64;

 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum PoolQuotaProbeMode {
@@ -214,6 +217,34 @@ fn pool_quota_probe_selection_limit_for_mode(
    }
 }

+fn pool_quota_probe_auto_interval_seconds(demand_snapshot: &ProviderPoolDemandSnapshot) -> u64 {
+    let live_pressure = u64::try_from(demand_snapshot.in_flight).unwrap_or(u64::MAX);
+    let ema_pressure =
+        if demand_snapshot.ema_in_flight.is_finite() && demand_snapshot.ema_in_flight > 0.0 {
+            demand_snapshot
+                .ema_in_flight
+                .ceil()
+                .clamp(0.0, POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE as f64) as u64
+        } else {
+            0
+        };
+    let request_pressure = live_pressure.max(ema_pressure);
+    if request_pressure == 0 {
+        return POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS;
+    }
+
+    let hot_pressure = u64::try_from(demand_snapshot.desired_hot).unwrap_or(u64::MAX);
+    let pressure = request_pressure
+        .max(hot_pressure)
+        .clamp(1, POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE);
+    POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
+        .saturating_div(pressure)
+        .clamp(
+            POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS,
+            POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS,
+        )
+}
+
 fn active_probe_member_remains_valid(score: Option<&StoredPoolMemberScore>) -> bool {
    match score.map(|score| score.hard_state) {
        Some(PoolMemberHardState::Available | PoolMemberHardState::Unknown) | None => true,
@@ -894,10 +925,7 @@ async fn select_keys_for_provider(
                            .get(key_id.as_str())
                            .is_none_or(|last_probe_ts| {
                                now_ts.saturating_sub(*last_probe_ts)
-                                    >= pool_config
-                                        .probing_interval_minutes
-                                        .clamp(1, 1440)
-                                        .saturating_mul(60)
+                                    >= pool_quota_probe_auto_interval_seconds(&demand_snapshot)
                            })
                    }
                    PoolQuotaProbeMode::Burst => {
@@ -925,10 +953,7 @@ async fn select_keys_for_provider(
        }

        let stamp_interval_seconds = match mode {
-            PoolQuotaProbeMode::Base => pool_config
-                .probing_interval_minutes
-                .clamp(1, 1440)
-                .saturating_mul(60),
+            PoolQuotaProbeMode::Base => pool_quota_probe_auto_interval_seconds(&demand_snapshot),
            PoolQuotaProbeMode::Burst => POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS,
        };
        mark_probe_timestamps(
@@ -1756,6 +1781,42 @@ mod tests {
        assert_eq!(pool_quota_probe_target_count(3, Some(80.0), Some(10)), 3);
    }

+    #[test]
+    fn pool_quota_probe_auto_interval_tracks_request_pressure() {
+        let idle = ProviderPoolDemandSnapshot {
+            in_flight: 0,
+            ema_in_flight: 0.0,
+            desired_hot: 2,
+            sampled_at_unix_ms: 0,
+        };
+        assert_eq!(
+            pool_quota_probe_auto_interval_seconds(&idle),
+            POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
+        );
+
+        let active = ProviderPoolDemandSnapshot {
+            in_flight: 1,
+            ema_in_flight: 1.0,
+            desired_hot: 2,
+            sampled_at_unix_ms: 0,
+        };
+        assert!(
+            pool_quota_probe_auto_interval_seconds(&active)
+                < POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
+        );
+
+        let saturated = ProviderPoolDemandSnapshot {
+            in_flight: 128,
+            ema_in_flight: 128.0,
+            desired_hot: 128,
+            sampled_at_unix_ms: 0,
+        };
+        assert_eq!(
+            pool_quota_probe_auto_interval_seconds(&saturated),
+            POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS
+        );
+    }
+
    #[test]
    fn selects_only_pool_out_keys_to_fill_active_probe_target() {
        let key_ids = vec![
--- a/apps/aether-gateway/src/state/core.rs
+++ b/apps/aether-gateway/src/state/core.rs
@@ -47,7 +47,6 @@ use crate::maintenance::spawn_gemini_file_mapping_cleanup_worker;
 use crate::maintenance::spawn_oauth_token_refresh_worker;
 use crate::maintenance::spawn_pending_cleanup_worker;
 use crate::maintenance::spawn_pool_monitor_worker;
-use crate::maintenance::spawn_pool_quota_probe_worker;
 use crate::maintenance::spawn_pool_score_rebuild_worker;
 use crate::maintenance::spawn_provider_checkin_worker;
 use crate::maintenance::spawn_proxy_node_metrics_cleanup_worker;
@@ -1173,10 +1172,6 @@ impl AppState {
            crate::task_runtime::TASK_KEY_POOL_MONITOR,
            spawn_pool_monitor_worker(self.data.clone()),
        );
-        supervise_worker(
-            crate::task_runtime::TASK_KEY_POOL_QUOTA_PROBE,
-            spawn_pool_quota_probe_worker(self.clone()),
-        );
        supervise_worker(
            crate::task_runtime::TASK_KEY_ACCOUNT_SELF_CHECK,
            spawn_account_self_check_worker(self.clone()),
--- a/apps/aether-gateway/src/task_runtime/mod.rs
+++ b/apps/aether-gateway/src/task_runtime/mod.rs
@@ -21,7 +21,6 @@ pub(crate) const TASK_KEY_USAGE_QUEUE_WORKER: &str = "usage.queue.worker";
 pub(crate) const TASK_KEY_VIDEO_TASK_POLLER: &str = "video.task.poller";
 pub(crate) const TASK_KEY_MODEL_FETCH_WORKER: &str = "model.fetch.worker";
 pub(crate) const TASK_KEY_PROVIDER_QUOTA_RESET: &str = "provider.quota.reset.worker";
-pub(crate) const TASK_KEY_POOL_QUOTA_PROBE: &str = "pool.quota.probe.worker";
 pub(crate) const TASK_KEY_ACCOUNT_SELF_CHECK: &str = "account.self_check.worker";
 pub(crate) const TASK_KEY_POOL_SCORE_REBUILD: &str = "pool.score.rebuild.worker";
 pub(crate) const TASK_KEY_POOL_MONITOR: &str = "pool.monitor.worker";
@@ -95,14 +94,6 @@ const TASK_DEFINITIONS: &[TaskDefinition] = &[
        true,
        RETRY_ONCE,
    ),
-    TaskDefinition::new(
-        TASK_KEY_POOL_QUOTA_PROBE,
-        TaskKind::Scheduled,
-        "interval",
-        true,
-        true,
-        RETRY_ONCE,
-    ),
    TaskDefinition::new(
        TASK_KEY_ACCOUNT_SELF_CHECK,
        TaskKind::Scheduled,
--- a/frontend/src/api/endpoints/types/provider.ts
+++ b/frontend/src/api/endpoints/types/provider.ts
@@ -583,7 +583,6 @@ export interface PoolAdvancedConfig {
  score_fallback_scan_limit?: number | null
  score_rules?: PoolScoreRules | null
  probing_enabled?: boolean
-  probing_interval_minutes?: number | null
  // deprecated: retained only for backward-compatible reads
  probing_target_percent?: number | null
  // deprecated: retained only for backward-compatible reads
--- a/frontend/src/features/pool/components/PoolAdvancedDialog.vue
+++ b/frontend/src/features/pool/components/PoolAdvancedDialog.vue
@@ -18,7 +18,7 @@
            </span>
          </div>
          <p class="text-xs leading-5 text-muted-foreground">
-            控制自动冷却、主动探测、异常清理和全局调度优先级。
+            控制自动冷却、自适应热池、异常清理和全局调度优先级。
          </p>
        </div>

@@ -67,28 +67,6 @@
          </div>
        </div>

-        <div
-          v-if="form.probing_enabled"
-          class="rounded-xl border border-dashed border-primary/25 bg-primary/5 p-4"
-        >
-          <div class="grid gap-3 sm:grid-cols-2">
-            <div class="space-y-1.5">
-              <Label>
-                探测间隔
-                <span class="text-xs text-muted-foreground">(分钟)</span>
-              </Label>
-              <Input
-                :model-value="form.probing_interval_minutes ?? ''"
-                type="number"
-                min="1"
-                max="1440"
-                placeholder="10"
-                @update:model-value="(v) => form.probing_interval_minutes = parseNum(v)"
-              />
-            </div>
-          </div>
-        </div>
-
        <div
          v-if="form.account_self_check_enabled"
          class="space-y-3 rounded-xl border border-dashed border-primary/25 bg-primary/5 p-4"
@@ -261,7 +239,7 @@
              </span>
            </div>
            <p class="text-xs leading-5 text-muted-foreground">
-              控制刷新 OAuth、主动探测和批量额度处理时的并行请求数。
+              控制刷新 OAuth、自适应热池和批量额度处理时的并行请求数。
            </p>
          </div>

@@ -708,7 +686,6 @@ const form = ref({
  request_failure_penalty: null as number | null | undefined,
  probe_failure_cooldown_threshold: null as number | null | undefined,
  probing_enabled: false,
-  probing_interval_minutes: null as number | null | undefined,
  account_self_check_enabled: false,
  account_self_check_interval_minutes: null as number | null | undefined,
  account_self_check_concurrency: null as number | null | undefined,
@@ -807,7 +784,6 @@ watch(() => props.modelValue, (open) => {
    request_failure_penalty: scoreRules?.request_failure_penalty ?? null,
    probe_failure_cooldown_threshold: scoreRules?.probe_failure_cooldown_threshold ?? null,
    probing_enabled: cfg?.probing_enabled ?? false,
-    probing_interval_minutes: cfg?.probing_interval_minutes ?? null,
    account_self_check_enabled: cfg?.account_self_check_enabled ?? false,
    account_self_check_interval_minutes: cfg?.account_self_check_interval_minutes ?? null,
    account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null,
@@ -855,6 +831,7 @@ async function handleSave() {
      'probing_active_target_count',
      'active_probe_target_percent',
      'active_probe_target_count',
+      'probing_interval_minutes',
      'account_self_check_method',
      'self_check_method',
      'account_self_check_request',
@@ -879,9 +856,6 @@ async function handleSave() {
      score_fallback_scan_limit: form.value.score_fallback_scan_limit ?? undefined,
      score_rules: scoreRules,
      probing_enabled: form.value.probing_enabled,
-      probing_interval_minutes: form.value.probing_enabled
-        ? (form.value.probing_interval_minutes ?? undefined)
-        : undefined,
      account_self_check_enabled: form.value.account_self_check_enabled,
      account_self_check_interval_minutes: form.value.account_self_check_enabled
        ? (form.value.account_self_check_interval_minutes ?? undefined)
--- a/frontend/src/features/pool/utils/tests/poolAdvancedDialog.spec.ts
+++ b/frontend/src/features/pool/utils/tests/poolAdvancedDialog.spec.ts
@@ -27,7 +27,7 @@ describe('poolAdvancedDialog', () => {
      },
      {
        key: 'probing_enabled',
-        label: '主动探测',
+        label: '自适应热池',
        description: '自动维护热池，缺口时异步补位。',
      },
      {
--- a/frontend/src/features/pool/utils/poolAdvancedDialog.ts
+++ b/frontend/src/features/pool/utils/poolAdvancedDialog.ts
@@ -34,7 +34,7 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] {
    },
    {
      key: 'probing_enabled',
-      label: '主动探测',
+      label: '自适应热池',
      description: '自动维护热池，缺口时异步补位。',
    },
    {