Make pool probing request-driven

This commit is contained in:
fawney19
2026-05-16 00:18:47 +08:00
parent 43d891bee1
commit d53546d56f
10 changed files with 78 additions and 69 deletions

View File

@@ -407,7 +407,6 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
overload_cooldown_seconds: 30, overload_cooldown_seconds: 30,
health_policy_enabled: true, health_policy_enabled: true,
probing_enabled: false, probing_enabled: false,
probing_interval_minutes: 10,
probing_target_percent: None, probing_target_percent: None,
probing_target_count: None, probing_target_count: None,
probe_concurrency: 4, probe_concurrency: 4,
@@ -475,12 +474,6 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
.get("probing_enabled") .get("probing_enabled")
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false), .unwrap_or(false),
probing_interval_minutes: pool_advanced
.get("probing_interval_minutes")
.and_then(json_u64)
.filter(|value| *value > 0)
.map(|value| value.min(1440))
.unwrap_or(10),
probing_target_percent: parse_pool_probe_target_percent(pool_advanced), probing_target_percent: parse_pool_probe_target_percent(pool_advanced),
probing_target_count: parse_pool_probe_target_count(pool_advanced), probing_target_count: parse_pool_probe_target_count(pool_advanced),
probe_concurrency: pool_advanced probe_concurrency: pool_advanced
@@ -592,7 +585,6 @@ mod tests {
"overload_cooldown_seconds": 45, "overload_cooldown_seconds": 45,
"health_policy_enabled": false, "health_policy_enabled": false,
"probing_enabled": true, "probing_enabled": true,
"probing_interval_minutes": 20,
"probing_target_percent": 25, "probing_target_percent": 25,
"probing_target_count": 3, "probing_target_count": 3,
"probe_concurrency": 6, "probe_concurrency": 6,
@@ -634,7 +626,6 @@ mod tests {
assert_eq!(config.overload_cooldown_seconds, 45); assert_eq!(config.overload_cooldown_seconds, 45);
assert!(!config.health_policy_enabled); assert!(!config.health_policy_enabled);
assert!(config.probing_enabled); assert!(config.probing_enabled);
assert_eq!(config.probing_interval_minutes, 20);
assert_eq!(config.probing_target_percent, Some(25.0)); assert_eq!(config.probing_target_percent, Some(25.0));
assert_eq!(config.probing_target_count, Some(3)); assert_eq!(config.probing_target_count, Some(3));
assert_eq!(config.probe_concurrency, 6); assert_eq!(config.probe_concurrency, 6);
@@ -656,7 +647,7 @@ mod tests {
} }
#[test] #[test]
fn clamps_pool_quota_probe_interval_to_python_range() { fn ignores_legacy_pool_quota_probe_interval() {
let provider = sample_provider(json!({ let provider = sample_provider(json!({
"pool_advanced": { "pool_advanced": {
"probing_enabled": true, "probing_enabled": true,
@@ -664,7 +655,7 @@ mod tests {
} }
})); }));
let config = admin_provider_pool_config(&provider).expect("pool config should exist"); let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert_eq!(config.probing_interval_minutes, 1440); assert!(config.probing_enabled);
let provider = sample_provider(json!({ let provider = sample_provider(json!({
"pool_advanced": { "pool_advanced": {
@@ -673,7 +664,7 @@ mod tests {
} }
})); }));
let config = admin_provider_pool_config(&provider).expect("pool config should exist"); let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert_eq!(config.probing_interval_minutes, 10); assert!(config.probing_enabled);
} }
#[test] #[test]

View File

@@ -614,7 +614,6 @@ mod tests {
overload_cooldown_seconds: 30, overload_cooldown_seconds: 30,
health_policy_enabled: true, health_policy_enabled: true,
probing_enabled: false, probing_enabled: false,
probing_interval_minutes: 10,
probing_target_percent: None, probing_target_percent: None,
probing_target_count: None, probing_target_count: None,
probe_concurrency: 4, probe_concurrency: 4,

View File

@@ -45,7 +45,6 @@ pub(crate) struct AdminProviderPoolConfig {
pub(crate) overload_cooldown_seconds: u64, pub(crate) overload_cooldown_seconds: u64,
pub(crate) health_policy_enabled: bool, pub(crate) health_policy_enabled: bool,
pub(crate) probing_enabled: bool, pub(crate) probing_enabled: bool,
pub(crate) probing_interval_minutes: u64,
pub(crate) probing_target_percent: Option<f64>, pub(crate) probing_target_percent: Option<f64>,
pub(crate) probing_target_count: Option<u64>, pub(crate) probing_target_count: Option<u64>,
pub(crate) probe_concurrency: u64, pub(crate) probe_concurrency: u64,

View File

@@ -29,7 +29,7 @@ use crate::handlers::shared::provider_pool::{
}; };
use crate::provider_pool_demand::{ use crate::provider_pool_demand::{
provider_pool_burst_pending_key, read_provider_pool_demand_snapshot, provider_pool_burst_pending_key, read_provider_pool_demand_snapshot,
sample_provider_pool_demand, sample_provider_pool_demand, ProviderPoolDemandSnapshot,
}; };
use super::pool_score_rebuild::ensure_provider_key_pool_scores_for_keys; use super::pool_score_rebuild::ensure_provider_key_pool_scores_for_keys;
@@ -44,6 +44,9 @@ const POOL_QUOTA_PROBE_BURST_TRIGGER_LOCK_TTL_MS: u64 = 30_000;
const POOL_QUOTA_PROBE_BURST_PENDING_PREFIX: &str = "ap:quota_probe:burst_pending"; const POOL_QUOTA_PROBE_BURST_PENDING_PREFIX: &str = "ap:quota_probe:burst_pending";
const POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS: u64 = 30; const POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS: u64 = 30;
const POOL_QUOTA_PROBE_BURST_RETRY_GUARD_SECONDS: u64 = 15; const POOL_QUOTA_PROBE_BURST_RETRY_GUARD_SECONDS: u64 = 15;
const POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS: u64 = 30;
const POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS: u64 = 10 * 60;
const POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE: u64 = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PoolQuotaProbeMode { enum PoolQuotaProbeMode {
@@ -214,6 +217,34 @@ fn pool_quota_probe_selection_limit_for_mode(
} }
} }
fn pool_quota_probe_auto_interval_seconds(demand_snapshot: &ProviderPoolDemandSnapshot) -> u64 {
let live_pressure = u64::try_from(demand_snapshot.in_flight).unwrap_or(u64::MAX);
let ema_pressure =
if demand_snapshot.ema_in_flight.is_finite() && demand_snapshot.ema_in_flight > 0.0 {
demand_snapshot
.ema_in_flight
.ceil()
.clamp(0.0, POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE as f64) as u64
} else {
0
};
let request_pressure = live_pressure.max(ema_pressure);
if request_pressure == 0 {
return POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS;
}
let hot_pressure = u64::try_from(demand_snapshot.desired_hot).unwrap_or(u64::MAX);
let pressure = request_pressure
.max(hot_pressure)
.clamp(1, POOL_QUOTA_PROBE_AUTO_MAX_PRESSURE);
POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
.saturating_div(pressure)
.clamp(
POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS,
POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS,
)
}
fn active_probe_member_remains_valid(score: Option<&StoredPoolMemberScore>) -> bool { fn active_probe_member_remains_valid(score: Option<&StoredPoolMemberScore>) -> bool {
match score.map(|score| score.hard_state) { match score.map(|score| score.hard_state) {
Some(PoolMemberHardState::Available | PoolMemberHardState::Unknown) | None => true, Some(PoolMemberHardState::Available | PoolMemberHardState::Unknown) | None => true,
@@ -894,10 +925,7 @@ async fn select_keys_for_provider(
.get(key_id.as_str()) .get(key_id.as_str())
.is_none_or(|last_probe_ts| { .is_none_or(|last_probe_ts| {
now_ts.saturating_sub(*last_probe_ts) now_ts.saturating_sub(*last_probe_ts)
>= pool_config >= pool_quota_probe_auto_interval_seconds(&demand_snapshot)
.probing_interval_minutes
.clamp(1, 1440)
.saturating_mul(60)
}) })
} }
PoolQuotaProbeMode::Burst => { PoolQuotaProbeMode::Burst => {
@@ -925,10 +953,7 @@ async fn select_keys_for_provider(
} }
let stamp_interval_seconds = match mode { let stamp_interval_seconds = match mode {
PoolQuotaProbeMode::Base => pool_config PoolQuotaProbeMode::Base => pool_quota_probe_auto_interval_seconds(&demand_snapshot),
.probing_interval_minutes
.clamp(1, 1440)
.saturating_mul(60),
PoolQuotaProbeMode::Burst => POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS, PoolQuotaProbeMode::Burst => POOL_QUOTA_PROBE_BURST_PENDING_TTL_SECONDS,
}; };
mark_probe_timestamps( mark_probe_timestamps(
@@ -1756,6 +1781,42 @@ mod tests {
assert_eq!(pool_quota_probe_target_count(3, Some(80.0), Some(10)), 3); assert_eq!(pool_quota_probe_target_count(3, Some(80.0), Some(10)), 3);
} }
#[test]
fn pool_quota_probe_auto_interval_tracks_request_pressure() {
let idle = ProviderPoolDemandSnapshot {
in_flight: 0,
ema_in_flight: 0.0,
desired_hot: 2,
sampled_at_unix_ms: 0,
};
assert_eq!(
pool_quota_probe_auto_interval_seconds(&idle),
POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
);
let active = ProviderPoolDemandSnapshot {
in_flight: 1,
ema_in_flight: 1.0,
desired_hot: 2,
sampled_at_unix_ms: 0,
};
assert!(
pool_quota_probe_auto_interval_seconds(&active)
< POOL_QUOTA_PROBE_AUTO_MAX_INTERVAL_SECONDS
);
let saturated = ProviderPoolDemandSnapshot {
in_flight: 128,
ema_in_flight: 128.0,
desired_hot: 128,
sampled_at_unix_ms: 0,
};
assert_eq!(
pool_quota_probe_auto_interval_seconds(&saturated),
POOL_QUOTA_PROBE_AUTO_MIN_INTERVAL_SECONDS
);
}
#[test] #[test]
fn selects_only_pool_out_keys_to_fill_active_probe_target() { fn selects_only_pool_out_keys_to_fill_active_probe_target() {
let key_ids = vec![ let key_ids = vec![

View File

@@ -47,7 +47,6 @@ use crate::maintenance::spawn_gemini_file_mapping_cleanup_worker;
use crate::maintenance::spawn_oauth_token_refresh_worker; use crate::maintenance::spawn_oauth_token_refresh_worker;
use crate::maintenance::spawn_pending_cleanup_worker; use crate::maintenance::spawn_pending_cleanup_worker;
use crate::maintenance::spawn_pool_monitor_worker; use crate::maintenance::spawn_pool_monitor_worker;
use crate::maintenance::spawn_pool_quota_probe_worker;
use crate::maintenance::spawn_pool_score_rebuild_worker; use crate::maintenance::spawn_pool_score_rebuild_worker;
use crate::maintenance::spawn_provider_checkin_worker; use crate::maintenance::spawn_provider_checkin_worker;
use crate::maintenance::spawn_proxy_node_metrics_cleanup_worker; use crate::maintenance::spawn_proxy_node_metrics_cleanup_worker;
@@ -1173,10 +1172,6 @@ impl AppState {
crate::task_runtime::TASK_KEY_POOL_MONITOR, crate::task_runtime::TASK_KEY_POOL_MONITOR,
spawn_pool_monitor_worker(self.data.clone()), spawn_pool_monitor_worker(self.data.clone()),
); );
supervise_worker(
crate::task_runtime::TASK_KEY_POOL_QUOTA_PROBE,
spawn_pool_quota_probe_worker(self.clone()),
);
supervise_worker( supervise_worker(
crate::task_runtime::TASK_KEY_ACCOUNT_SELF_CHECK, crate::task_runtime::TASK_KEY_ACCOUNT_SELF_CHECK,
spawn_account_self_check_worker(self.clone()), spawn_account_self_check_worker(self.clone()),

View File

@@ -21,7 +21,6 @@ pub(crate) const TASK_KEY_USAGE_QUEUE_WORKER: &str = "usage.queue.worker";
pub(crate) const TASK_KEY_VIDEO_TASK_POLLER: &str = "video.task.poller"; pub(crate) const TASK_KEY_VIDEO_TASK_POLLER: &str = "video.task.poller";
pub(crate) const TASK_KEY_MODEL_FETCH_WORKER: &str = "model.fetch.worker"; pub(crate) const TASK_KEY_MODEL_FETCH_WORKER: &str = "model.fetch.worker";
pub(crate) const TASK_KEY_PROVIDER_QUOTA_RESET: &str = "provider.quota.reset.worker"; pub(crate) const TASK_KEY_PROVIDER_QUOTA_RESET: &str = "provider.quota.reset.worker";
pub(crate) const TASK_KEY_POOL_QUOTA_PROBE: &str = "pool.quota.probe.worker";
pub(crate) const TASK_KEY_ACCOUNT_SELF_CHECK: &str = "account.self_check.worker"; pub(crate) const TASK_KEY_ACCOUNT_SELF_CHECK: &str = "account.self_check.worker";
pub(crate) const TASK_KEY_POOL_SCORE_REBUILD: &str = "pool.score.rebuild.worker"; pub(crate) const TASK_KEY_POOL_SCORE_REBUILD: &str = "pool.score.rebuild.worker";
pub(crate) const TASK_KEY_POOL_MONITOR: &str = "pool.monitor.worker"; pub(crate) const TASK_KEY_POOL_MONITOR: &str = "pool.monitor.worker";
@@ -95,14 +94,6 @@ const TASK_DEFINITIONS: &[TaskDefinition] = &[
true, true,
RETRY_ONCE, RETRY_ONCE,
), ),
TaskDefinition::new(
TASK_KEY_POOL_QUOTA_PROBE,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new( TaskDefinition::new(
TASK_KEY_ACCOUNT_SELF_CHECK, TASK_KEY_ACCOUNT_SELF_CHECK,
TaskKind::Scheduled, TaskKind::Scheduled,

View File

@@ -583,7 +583,6 @@ export interface PoolAdvancedConfig {
score_fallback_scan_limit?: number | null score_fallback_scan_limit?: number | null
score_rules?: PoolScoreRules | null score_rules?: PoolScoreRules | null
probing_enabled?: boolean probing_enabled?: boolean
probing_interval_minutes?: number | null
// deprecated: retained only for backward-compatible reads // deprecated: retained only for backward-compatible reads
probing_target_percent?: number | null probing_target_percent?: number | null
// deprecated: retained only for backward-compatible reads // deprecated: retained only for backward-compatible reads

View File

@@ -18,7 +18,7 @@
</span> </span>
</div> </div>
<p class="text-xs leading-5 text-muted-foreground"> <p class="text-xs leading-5 text-muted-foreground">
控制自动冷却主动探测异常清理和全局调度优先级 控制自动冷却自适应热池异常清理和全局调度优先级
</p> </p>
</div> </div>
@@ -67,28 +67,6 @@
</div> </div>
</div> </div>
<div
v-if="form.probing_enabled"
class="rounded-xl border border-dashed border-primary/25 bg-primary/5 p-4"
>
<div class="grid gap-3 sm:grid-cols-2">
<div class="space-y-1.5">
<Label>
探测间隔
<span class="text-xs text-muted-foreground">(分钟)</span>
</Label>
<Input
:model-value="form.probing_interval_minutes ?? ''"
type="number"
min="1"
max="1440"
placeholder="10"
@update:model-value="(v) => form.probing_interval_minutes = parseNum(v)"
/>
</div>
</div>
</div>
<div <div
v-if="form.account_self_check_enabled" v-if="form.account_self_check_enabled"
class="space-y-3 rounded-xl border border-dashed border-primary/25 bg-primary/5 p-4" class="space-y-3 rounded-xl border border-dashed border-primary/25 bg-primary/5 p-4"
@@ -261,7 +239,7 @@
</span> </span>
</div> </div>
<p class="text-xs leading-5 text-muted-foreground"> <p class="text-xs leading-5 text-muted-foreground">
控制刷新 OAuth、主动探测和批量额度处理时的并行请求数。 控制刷新 OAuth、自适应热池和批量额度处理时的并行请求数。
</p> </p>
</div> </div>
@@ -708,7 +686,6 @@ const form = ref({
request_failure_penalty: null as number | null | undefined, request_failure_penalty: null as number | null | undefined,
probe_failure_cooldown_threshold: null as number | null | undefined, probe_failure_cooldown_threshold: null as number | null | undefined,
probing_enabled: false, probing_enabled: false,
probing_interval_minutes: null as number | null | undefined,
account_self_check_enabled: false, account_self_check_enabled: false,
account_self_check_interval_minutes: null as number | null | undefined, account_self_check_interval_minutes: null as number | null | undefined,
account_self_check_concurrency: null as number | null | undefined, account_self_check_concurrency: null as number | null | undefined,
@@ -807,7 +784,6 @@ watch(() => props.modelValue, (open) => {
request_failure_penalty: scoreRules?.request_failure_penalty ?? null, request_failure_penalty: scoreRules?.request_failure_penalty ?? null,
probe_failure_cooldown_threshold: scoreRules?.probe_failure_cooldown_threshold ?? null, probe_failure_cooldown_threshold: scoreRules?.probe_failure_cooldown_threshold ?? null,
probing_enabled: cfg?.probing_enabled ?? false, probing_enabled: cfg?.probing_enabled ?? false,
probing_interval_minutes: cfg?.probing_interval_minutes ?? null,
account_self_check_enabled: cfg?.account_self_check_enabled ?? false, account_self_check_enabled: cfg?.account_self_check_enabled ?? false,
account_self_check_interval_minutes: cfg?.account_self_check_interval_minutes ?? null, account_self_check_interval_minutes: cfg?.account_self_check_interval_minutes ?? null,
account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null, account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null,
@@ -855,6 +831,7 @@ async function handleSave() {
'probing_active_target_count', 'probing_active_target_count',
'active_probe_target_percent', 'active_probe_target_percent',
'active_probe_target_count', 'active_probe_target_count',
'probing_interval_minutes',
'account_self_check_method', 'account_self_check_method',
'self_check_method', 'self_check_method',
'account_self_check_request', 'account_self_check_request',
@@ -879,9 +856,6 @@ async function handleSave() {
score_fallback_scan_limit: form.value.score_fallback_scan_limit ?? undefined, score_fallback_scan_limit: form.value.score_fallback_scan_limit ?? undefined,
score_rules: scoreRules, score_rules: scoreRules,
probing_enabled: form.value.probing_enabled, probing_enabled: form.value.probing_enabled,
probing_interval_minutes: form.value.probing_enabled
? (form.value.probing_interval_minutes ?? undefined)
: undefined,
account_self_check_enabled: form.value.account_self_check_enabled, account_self_check_enabled: form.value.account_self_check_enabled,
account_self_check_interval_minutes: form.value.account_self_check_enabled account_self_check_interval_minutes: form.value.account_self_check_enabled
? (form.value.account_self_check_interval_minutes ?? undefined) ? (form.value.account_self_check_interval_minutes ?? undefined)

View File

@@ -27,7 +27,7 @@ describe('poolAdvancedDialog', () => {
}, },
{ {
key: 'probing_enabled', key: 'probing_enabled',
label: '主动探测', label: '自适应热池',
description: '自动维护热池,缺口时异步补位。', description: '自动维护热池,缺口时异步补位。',
}, },
{ {

View File

@@ -34,7 +34,7 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] {
}, },
{ {
key: 'probing_enabled', key: 'probing_enabled',
label: '主动探测', label: '自适应热池',
description: '自动维护热池,缺口时异步补位。', description: '自动维护热池,缺口时异步补位。',
}, },
{ {