mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 00:17:45 +08:00
feat(codex): add optional minimum quota reserve for pool scheduling
This commit is contained in:
@@ -131,8 +131,13 @@ async fn schedule_pool_page_candidates(
|
|||||||
entry.1.insert(candidate.candidate.key_id.clone());
|
entry.1.insert(candidate.candidate.key_id.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
let key_context_by_id =
|
let key_context_by_id = read_pool_catalog_key_contexts_by_id(
|
||||||
read_pool_catalog_key_contexts_by_id(state, &candidates, provider_model_name).await;
|
state,
|
||||||
|
&candidates,
|
||||||
|
provider_model_name,
|
||||||
|
effective_pool_config,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let mut runtime_by_provider = BTreeMap::new();
|
let mut runtime_by_provider = BTreeMap::new();
|
||||||
let mut pool_config_by_provider = BTreeMap::new();
|
let mut pool_config_by_provider = BTreeMap::new();
|
||||||
@@ -1029,6 +1034,26 @@ impl<'a> PoolKeyCursor<'a> {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if pool_config.reserve_minimum_quota
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
|
&key,
|
||||||
|
self.group.candidate.provider_type.as_str(),
|
||||||
|
Some(self.group.candidate.selected_provider_model_name.as_str()),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
self.seen_key_ids.insert(key.id.clone());
|
||||||
|
self.record_skip_reason(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON);
|
||||||
|
self.skipped_candidates
|
||||||
|
.push(SkippedLocalExecutionCandidate {
|
||||||
|
candidate: pool_candidate_from_catalog_key(&self.group, key),
|
||||||
|
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
|
||||||
|
transport: None,
|
||||||
|
ranking: self.group.ranking.clone(),
|
||||||
|
extra_data: None,
|
||||||
|
});
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
let candidate = pool_candidate_from_catalog_key(&self.group, key);
|
let candidate = pool_candidate_from_catalog_key(&self.group, key);
|
||||||
self.build_eligible_candidate(candidate).await
|
self.build_eligible_candidate(candidate).await
|
||||||
}
|
}
|
||||||
@@ -1429,15 +1454,23 @@ async fn read_pool_catalog_key_contexts_by_id(
|
|||||||
state: PlannerAppState<'_>,
|
state: PlannerAppState<'_>,
|
||||||
candidates: &[EligibleLocalExecutionCandidate],
|
candidates: &[EligibleLocalExecutionCandidate],
|
||||||
provider_model_name: Option<&str>,
|
provider_model_name: Option<&str>,
|
||||||
|
effective_pool_config: Option<&AdminProviderPoolConfig>,
|
||||||
) -> BTreeMap<String, PoolCatalogKeyContext> {
|
) -> BTreeMap<String, PoolCatalogKeyContext> {
|
||||||
let mut key_ids = Vec::new();
|
let mut key_ids = Vec::new();
|
||||||
let mut provider_type_by_key_id = BTreeMap::<String, String>::new();
|
let mut provider_type_by_key_id = BTreeMap::<String, String>::new();
|
||||||
|
let mut reserve_minimum_quota_key_ids = BTreeSet::new();
|
||||||
|
|
||||||
for candidate in candidates {
|
for candidate in candidates {
|
||||||
if pool_config_for_candidate(candidate).is_none() {
|
let Some(pool_config) = effective_pool_config
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| pool_config_for_candidate(candidate))
|
||||||
|
else {
|
||||||
continue;
|
continue;
|
||||||
}
|
};
|
||||||
let key_id = candidate.candidate.key_id.clone();
|
let key_id = candidate.candidate.key_id.clone();
|
||||||
|
if pool_config.reserve_minimum_quota {
|
||||||
|
reserve_minimum_quota_key_ids.insert(key_id.clone());
|
||||||
|
}
|
||||||
if let Entry::Vacant(entry) = provider_type_by_key_id.entry(key_id.clone()) {
|
if let Entry::Vacant(entry) = provider_type_by_key_id.entry(key_id.clone()) {
|
||||||
entry.insert(candidate.transport.provider.provider_type.clone());
|
entry.insert(candidate.transport.provider.provider_type.clone());
|
||||||
key_ids.push(key_id);
|
key_ids.push(key_id);
|
||||||
@@ -1489,16 +1522,20 @@ async fn read_pool_catalog_key_contexts_by_id(
|
|||||||
.get(&key.id)
|
.get(&key.id)
|
||||||
.map(String::as_str)
|
.map(String::as_str)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
(
|
let mut context = build_pool_catalog_key_context(
|
||||||
key.id.clone(),
|
state,
|
||||||
build_pool_catalog_key_context(
|
&provider_pool_service,
|
||||||
state,
|
&key,
|
||||||
&provider_pool_service,
|
provider_type,
|
||||||
|
provider_model_name,
|
||||||
|
);
|
||||||
|
context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id)
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
&key,
|
&key,
|
||||||
provider_type,
|
provider_type,
|
||||||
provider_model_name,
|
provider_model_name,
|
||||||
),
|
);
|
||||||
)
|
(key.id.clone(), context)
|
||||||
})
|
})
|
||||||
.collect::<BTreeMap<_, _>>();
|
.collect::<BTreeMap<_, _>>();
|
||||||
// A key can disappear between the candidate-row and catalog reads. Keep
|
// A key can disappear between the candidate-row and catalog reads. Keep
|
||||||
@@ -3964,6 +4001,91 @@ mod tests {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
|
||||||
|
for reserve_enabled in [false, true] {
|
||||||
|
for sticky in [false, true] {
|
||||||
|
for used_percent in [99.0, 98.0] {
|
||||||
|
let provider_config = Some(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"reserve_minimum_quota": reserve_enabled,
|
||||||
|
"skip_exhausted_accounts": false
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let provider =
|
||||||
|
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
|
||||||
|
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
|
||||||
|
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
|
||||||
|
reserved.upstream_metadata = Some(json!({
|
||||||
|
"codex": {"primary_used_percent": used_percent}
|
||||||
|
}));
|
||||||
|
let ready = sample_codex_pool_key("provider-pool", "key-ready");
|
||||||
|
let rows = vec![
|
||||||
|
sample_codex_pool_row("provider-pool", "endpoint-1", "key-low", 0),
|
||||||
|
sample_codex_pool_row("provider-pool", "endpoint-1", "key-ready", 0),
|
||||||
|
];
|
||||||
|
let data_state = GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||||
|
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider], vec![endpoint], vec![reserved, ready],
|
||||||
|
)),
|
||||||
|
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||||
|
)
|
||||||
|
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let group =
|
||||||
|
sample_codex_pool_group("provider-pool", "endpoint-1", 0, provider_config);
|
||||||
|
let pool_config =
|
||||||
|
pool_config_for_candidate(&group).expect("pool config should parse");
|
||||||
|
let sticky_token = sticky.then_some("reserve-session");
|
||||||
|
if sticky {
|
||||||
|
record_admin_provider_pool_success(
|
||||||
|
app.runtime_state.as_ref(),
|
||||||
|
"provider-pool",
|
||||||
|
"key-low",
|
||||||
|
&pool_config,
|
||||||
|
sticky_token,
|
||||||
|
0,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
let mut cursor = PoolKeyCursor::new(
|
||||||
|
PlannerAppState::new(&app),
|
||||||
|
group,
|
||||||
|
sticky_token,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
cursor.window_size = 1;
|
||||||
|
cursor.page_size = 1;
|
||||||
|
let mut returned = Vec::new();
|
||||||
|
while let Some(candidate) = cursor.next_key().await {
|
||||||
|
returned.push(candidate.candidate.key_id);
|
||||||
|
}
|
||||||
|
let reserve_reached = reserve_enabled && used_percent >= 99.0;
|
||||||
|
assert_eq!(
|
||||||
|
returned.contains(&"key-low".to_string()),
|
||||||
|
!reserve_reached,
|
||||||
|
"reserve={reserve_enabled}, sticky={sticky}, used={used_percent}"
|
||||||
|
);
|
||||||
|
assert!(returned.contains(&"key-ready".to_string()));
|
||||||
|
if reserve_reached {
|
||||||
|
assert_eq!(
|
||||||
|
cursor
|
||||||
|
.skip_reason_counts
|
||||||
|
.get(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
} else if sticky {
|
||||||
|
assert_eq!(returned.first().map(String::as_str), Some("key-low"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn pool_key_cursor_does_not_spend_effective_scan_budget_on_exhausted_accounts() {
|
async fn pool_key_cursor_does_not_spend_effective_scan_budget_on_exhausted_accounts() {
|
||||||
let provider_config = Some(json!({
|
let provider_config = Some(json!({
|
||||||
|
|||||||
@@ -411,6 +411,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
|||||||
unschedulable_rules: Vec::new(),
|
unschedulable_rules: Vec::new(),
|
||||||
lru_enabled: false,
|
lru_enabled: false,
|
||||||
skip_exhausted_accounts: false,
|
skip_exhausted_accounts: false,
|
||||||
|
reserve_minimum_quota: false,
|
||||||
sticky_session_ttl_seconds: 3600,
|
sticky_session_ttl_seconds: 3600,
|
||||||
latency_window_seconds: 3600,
|
latency_window_seconds: 3600,
|
||||||
latency_sample_limit: 50,
|
latency_sample_limit: 50,
|
||||||
@@ -446,6 +447,10 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
|||||||
.get("skip_exhausted_accounts")
|
.get("skip_exhausted_accounts")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
|
reserve_minimum_quota: pool_advanced
|
||||||
|
.get("reserve_minimum_quota")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false),
|
||||||
sticky_session_ttl_seconds: pool_advanced
|
sticky_session_ttl_seconds: pool_advanced
|
||||||
.get("sticky_session_ttl_seconds")
|
.get("sticky_session_ttl_seconds")
|
||||||
.and_then(json_u64)
|
.and_then(json_u64)
|
||||||
@@ -574,6 +579,22 @@ mod tests {
|
|||||||
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
||||||
|
|
||||||
assert!(!config.skip_exhausted_accounts);
|
assert!(!config.skip_exhausted_accounts);
|
||||||
|
assert!(!config.reserve_minimum_quota);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_reserve_minimum_quota_independently_of_skip_exhausted_accounts() {
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let provider = sample_provider(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"reserve_minimum_quota": enabled,
|
||||||
|
"skip_exhausted_accounts": false
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
||||||
|
assert_eq!(config.reserve_minimum_quota, enabled);
|
||||||
|
assert!(!config.skip_exhausted_accounts);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -651,6 +651,7 @@ mod tests {
|
|||||||
unschedulable_rules: Vec::new(),
|
unschedulable_rules: Vec::new(),
|
||||||
lru_enabled: true,
|
lru_enabled: true,
|
||||||
skip_exhausted_accounts: false,
|
skip_exhausted_accounts: false,
|
||||||
|
reserve_minimum_quota: false,
|
||||||
sticky_session_ttl_seconds: 120,
|
sticky_session_ttl_seconds: 120,
|
||||||
latency_window_seconds: 600,
|
latency_window_seconds: 600,
|
||||||
latency_sample_limit: 10,
|
latency_sample_limit: 10,
|
||||||
|
|||||||
@@ -1117,10 +1117,16 @@ pub(super) fn build_admin_pool_key_payload(
|
|||||||
let health_score = admin_pool_health_score(key);
|
let health_score = admin_pool_health_score(key);
|
||||||
let circuit_breaker_open = false;
|
let circuit_breaker_open = false;
|
||||||
let auth_semantics = provider_key_auth_semantics(key, provider_type);
|
let auth_semantics = provider_key_auth_semantics(key, provider_type);
|
||||||
let account_quota_exhausted = pool_config
|
let account_quota_exhausted = pool_config.as_ref().is_some_and(|config| {
|
||||||
.as_ref()
|
(config.skip_exhausted_accounts
|
||||||
.is_some_and(|config| config.skip_exhausted_accounts)
|
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type))
|
||||||
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
|
|| (config.reserve_minimum_quota
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
|
key,
|
||||||
|
provider_type,
|
||||||
|
None,
|
||||||
|
))
|
||||||
|
});
|
||||||
let auth_config = state.parse_catalog_auth_config_json(key);
|
let auth_config = state.parse_catalog_auth_config_json(key);
|
||||||
let oauth_expires_at =
|
let oauth_expires_at =
|
||||||
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
|
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
|
||||||
|
|||||||
@@ -488,9 +488,16 @@ pub(super) fn admin_pool_key_visible_status_filter(
|
|||||||
) {
|
) {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
if pool_config.is_some_and(|config| config.skip_exhausted_accounts)
|
if pool_config.is_some_and(|config| {
|
||||||
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type)
|
(config.skip_exhausted_accounts
|
||||||
{
|
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type))
|
||||||
|
|| (config.reserve_minimum_quota
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
|
key,
|
||||||
|
provider_type,
|
||||||
|
None,
|
||||||
|
))
|
||||||
|
}) {
|
||||||
return "quota_exhausted";
|
return "quota_exhausted";
|
||||||
}
|
}
|
||||||
if !key.is_active {
|
if !key.is_active {
|
||||||
|
|||||||
@@ -1340,6 +1340,7 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
|
|||||||
provider.id.clone(),
|
provider.id.clone(),
|
||||||
provider_query_ai_pool_runtime_state(&runtime),
|
provider_query_ai_pool_runtime_state(&runtime),
|
||||||
);
|
);
|
||||||
|
let reserve_minimum_quota = pool_config.reserve_minimum_quota;
|
||||||
let pool_config =
|
let pool_config =
|
||||||
provider_query_ai_pool_scheduling_config(pool_config, provider.provider_type.as_str());
|
provider_query_ai_pool_scheduling_config(pool_config, provider.provider_type.as_str());
|
||||||
let inputs = keys
|
let inputs = keys
|
||||||
@@ -1351,6 +1352,14 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
|
|||||||
effective_model: effective_model.to_string(),
|
effective_model: effective_model.to_string(),
|
||||||
scheduler_skip_reason: None,
|
scheduler_skip_reason: None,
|
||||||
};
|
};
|
||||||
|
let mut key_context =
|
||||||
|
provider_query_pool_catalog_key_context(state, &key, &provider.provider_type);
|
||||||
|
key_context.quota_exhausted |= reserve_minimum_quota
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
|
&key,
|
||||||
|
&provider.provider_type,
|
||||||
|
Some(effective_model),
|
||||||
|
);
|
||||||
AiPoolCandidateInput {
|
AiPoolCandidateInput {
|
||||||
facts: AiPoolCandidateFacts {
|
facts: AiPoolCandidateFacts {
|
||||||
provider_id: provider.id.clone(),
|
provider_id: provider.id.clone(),
|
||||||
@@ -1362,11 +1371,7 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
|
|||||||
key_internal_priority: key.internal_priority,
|
key_internal_priority: key.internal_priority,
|
||||||
},
|
},
|
||||||
pool_config: Some(pool_config.clone()),
|
pool_config: Some(pool_config.clone()),
|
||||||
key_context: provider_query_pool_catalog_key_context(
|
key_context,
|
||||||
state,
|
|
||||||
&key,
|
|
||||||
&provider.provider_type,
|
|
||||||
),
|
|
||||||
candidate,
|
candidate,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ pub(crate) struct AdminProviderPoolConfig {
|
|||||||
pub(crate) unschedulable_rules: Vec<AdminProviderPoolUnschedulableRule>,
|
pub(crate) unschedulable_rules: Vec<AdminProviderPoolUnschedulableRule>,
|
||||||
pub(crate) lru_enabled: bool,
|
pub(crate) lru_enabled: bool,
|
||||||
pub(crate) skip_exhausted_accounts: bool,
|
pub(crate) skip_exhausted_accounts: bool,
|
||||||
|
pub(crate) reserve_minimum_quota: bool,
|
||||||
pub(crate) sticky_session_ttl_seconds: u64,
|
pub(crate) sticky_session_ttl_seconds: u64,
|
||||||
pub(crate) latency_window_seconds: u64,
|
pub(crate) latency_window_seconds: u64,
|
||||||
pub(crate) latency_sample_limit: u64,
|
pub(crate) latency_sample_limit: u64,
|
||||||
|
|||||||
@@ -40,10 +40,6 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
|
|||||||
) -> Result<CandidateRuntimeSelectionSnapshot, GatewayError> {
|
) -> Result<CandidateRuntimeSelectionSnapshot, GatewayError> {
|
||||||
let provider_concurrent_limits = read_provider_concurrent_limits(state, candidates).await?;
|
let provider_concurrent_limits = read_provider_concurrent_limits(state, candidates).await?;
|
||||||
let provider_pool_state = read_provider_pool_state_map(state, candidates).await?;
|
let provider_pool_state = read_provider_pool_state_map(state, candidates).await?;
|
||||||
let provider_skip_exhausted_accounts = provider_pool_state
|
|
||||||
.iter()
|
|
||||||
.map(|(provider_id, state)| (provider_id.clone(), state.skip_exhausted_accounts))
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let pool_provider_ids = provider_pool_state
|
let pool_provider_ids = provider_pool_state
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|(provider_id, state)| state.pool_enabled.then_some(provider_id.clone()))
|
.filter_map(|(provider_id, state)| state.pool_enabled.then_some(provider_id.clone()))
|
||||||
@@ -62,7 +58,7 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
|
|||||||
let key_account_quota_exhausted = read_key_account_quota_exhaustion_map(
|
let key_account_quota_exhausted = read_key_account_quota_exhaustion_map(
|
||||||
candidates,
|
candidates,
|
||||||
&provider_key_rpm_states,
|
&provider_key_rpm_states,
|
||||||
&provider_skip_exhausted_accounts,
|
&provider_pool_state,
|
||||||
);
|
);
|
||||||
let key_oauth_invalid =
|
let key_oauth_invalid =
|
||||||
read_key_oauth_invalid_map(candidates, &provider_key_rpm_states, now_unix_secs);
|
read_key_oauth_invalid_map(candidates, &provider_key_rpm_states, now_unix_secs);
|
||||||
@@ -360,6 +356,7 @@ async fn read_provider_quota_block_map(
|
|||||||
struct ProviderPoolState {
|
struct ProviderPoolState {
|
||||||
pool_enabled: bool,
|
pool_enabled: bool,
|
||||||
skip_exhausted_accounts: bool,
|
skip_exhausted_accounts: bool,
|
||||||
|
reserve_minimum_quota: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_provider_pool_state_map(
|
async fn read_provider_pool_state_map(
|
||||||
@@ -391,11 +388,17 @@ async fn read_provider_pool_state_map(
|
|||||||
.and_then(|value| value.get("skip_exhausted_accounts"))
|
.and_then(|value| value.get("skip_exhausted_accounts"))
|
||||||
.and_then(serde_json::Value::as_bool)
|
.and_then(serde_json::Value::as_bool)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let reserve_minimum_quota = pool_advanced
|
||||||
|
.and_then(serde_json::Value::as_object)
|
||||||
|
.and_then(|value| value.get("reserve_minimum_quota"))
|
||||||
|
.and_then(serde_json::Value::as_bool)
|
||||||
|
.unwrap_or(false);
|
||||||
(
|
(
|
||||||
provider.id,
|
provider.id,
|
||||||
ProviderPoolState {
|
ProviderPoolState {
|
||||||
pool_enabled: pool_advanced.is_some(),
|
pool_enabled: pool_advanced.is_some(),
|
||||||
skip_exhausted_accounts,
|
skip_exhausted_accounts,
|
||||||
|
reserve_minimum_quota,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -405,7 +408,7 @@ async fn read_provider_pool_state_map(
|
|||||||
fn read_key_account_quota_exhaustion_map(
|
fn read_key_account_quota_exhaustion_map(
|
||||||
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
|
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
|
||||||
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
|
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
|
||||||
provider_skip_exhausted_accounts: &BTreeMap<String, bool>,
|
provider_pool_state: &BTreeMap<String, ProviderPoolState>,
|
||||||
) -> BTreeMap<String, bool> {
|
) -> BTreeMap<String, bool> {
|
||||||
candidates
|
candidates
|
||||||
.iter()
|
.iter()
|
||||||
@@ -431,11 +434,19 @@ fn read_key_account_quota_exhaustion_map(
|
|||||||
candidate.provider_type.as_str(),
|
candidate.provider_type.as_str(),
|
||||||
candidate.selected_provider_model_name.as_str(),
|
candidate.selected_provider_model_name.as_str(),
|
||||||
);
|
);
|
||||||
let skip_configured = provider_skip_exhausted_accounts
|
let pool_state = provider_pool_state
|
||||||
.get(candidate.provider_id.as_str())
|
.get(candidate.provider_id.as_str())
|
||||||
.copied()
|
.copied()
|
||||||
.unwrap_or(false);
|
.unwrap_or_default();
|
||||||
hard_blocked || (skip_configured && account_exhausted)
|
let reserve_reached = pool_state.reserve_minimum_quota
|
||||||
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
|
key,
|
||||||
|
candidate.provider_type.as_str(),
|
||||||
|
Some(candidate.selected_provider_model_name.as_str()),
|
||||||
|
);
|
||||||
|
hard_blocked
|
||||||
|
|| reserve_reached
|
||||||
|
|| (pool_state.skip_exhausted_accounts && account_exhausted)
|
||||||
});
|
});
|
||||||
(candidate.key_id.clone(), exhausted)
|
(candidate.key_id.clone(), exhausted)
|
||||||
})
|
})
|
||||||
@@ -566,3 +577,64 @@ fn read_provider_key_rpm_reset_at_map(
|
|||||||
})
|
})
|
||||||
.collect::<BTreeMap<_, _>>()
|
.collect::<BTreeMap<_, _>>()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod reserve_minimum_quota_tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reserve_minimum_quota_is_independent_of_skip_exhausted_accounts() {
|
||||||
|
let candidate = SchedulerMinimalCandidateSelectionCandidate {
|
||||||
|
provider_id: "provider-codex".to_string(),
|
||||||
|
provider_name: "codex".to_string(),
|
||||||
|
provider_type: "codex".to_string(),
|
||||||
|
provider_priority: 0,
|
||||||
|
endpoint_id: "endpoint-codex".to_string(),
|
||||||
|
endpoint_api_format: "openai:responses".to_string(),
|
||||||
|
key_id: "key-codex".to_string(),
|
||||||
|
key_name: "codex".to_string(),
|
||||||
|
key_auth_type: "oauth".to_string(),
|
||||||
|
key_internal_priority: 0,
|
||||||
|
key_global_priority_for_format: None,
|
||||||
|
key_capabilities: None,
|
||||||
|
model_id: "model-codex".to_string(),
|
||||||
|
global_model_id: "global-model-codex".to_string(),
|
||||||
|
global_model_name: "gpt-5".to_string(),
|
||||||
|
selected_provider_model_name: "gpt-5".to_string(),
|
||||||
|
supports_streaming: true,
|
||||||
|
mapping_matched_model: None,
|
||||||
|
};
|
||||||
|
let mut key = StoredProviderCatalogKey::new(
|
||||||
|
candidate.key_id.clone(),
|
||||||
|
candidate.provider_id.clone(),
|
||||||
|
"codex".to_string(),
|
||||||
|
"oauth".to_string(),
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("key should build");
|
||||||
|
for reserve_enabled in [false, true] {
|
||||||
|
for used_percent in [99.0, 98.0] {
|
||||||
|
key.upstream_metadata =
|
||||||
|
Some(json!({"codex": {"primary_used_percent": used_percent}}));
|
||||||
|
let exhausted = read_key_account_quota_exhaustion_map(
|
||||||
|
std::slice::from_ref(&candidate),
|
||||||
|
&BTreeMap::from([(key.id.clone(), key.clone())]),
|
||||||
|
&BTreeMap::from([(
|
||||||
|
candidate.provider_id.clone(),
|
||||||
|
ProviderPoolState {
|
||||||
|
pool_enabled: true,
|
||||||
|
reserve_minimum_quota: reserve_enabled,
|
||||||
|
skip_exhausted_accounts: false,
|
||||||
|
},
|
||||||
|
)]),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
exhausted.get(&key.id),
|
||||||
|
Some(&(reserve_enabled && used_percent >= 99.0))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2364,6 +2364,78 @@ async fn gateway_marks_exhausted_codex_pool_key_as_blocked_when_flag_enabled() {
|
|||||||
assert_eq!(keys[0]["account_quota"], json!("5H剩余 0.0%"));
|
assert_eq!(keys[0]["account_quota"], json!("5H剩余 0.0%"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() {
|
||||||
|
for (reserve_enabled, used_percent, exhausted) in [
|
||||||
|
(true, 99.0, true),
|
||||||
|
(false, 99.0, false),
|
||||||
|
(true, 98.0, false),
|
||||||
|
] {
|
||||||
|
let mut provider = sample_provider("provider-codex", "codex", 10);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
provider.config = Some(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"reserve_minimum_quota": reserve_enabled,
|
||||||
|
"skip_exhausted_accounts": false,
|
||||||
|
"auto_remove_quota_exhausted_keys": true
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let mut key = sample_key(
|
||||||
|
"key-codex-reserved",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:responses",
|
||||||
|
"oauth-placeholder",
|
||||||
|
);
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.upstream_metadata = Some(json!({
|
||||||
|
"codex": {"primary_used_percent": used_percent}
|
||||||
|
}));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
|
||||||
|
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
Vec::new(),
|
||||||
|
vec![key],
|
||||||
|
)),
|
||||||
|
));
|
||||||
|
for status in ["all", "quota_exhausted", "available"] {
|
||||||
|
let response = local_admin_pool_response(
|
||||||
|
&state,
|
||||||
|
http::Method::GET,
|
||||||
|
&format!("/api/admin/pool/provider-codex/keys?page=1&page_size=50&status={status}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(
|
||||||
|
&to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("json body should parse");
|
||||||
|
let keys = payload["keys"].as_array().expect("keys should be array");
|
||||||
|
let visible = status == "all" || (status == "quota_exhausted") == exhausted;
|
||||||
|
assert_eq!(
|
||||||
|
keys.len(),
|
||||||
|
usize::from(visible),
|
||||||
|
"reserve={reserve_enabled}, used={used_percent}, status={status}"
|
||||||
|
);
|
||||||
|
if visible {
|
||||||
|
assert_eq!(keys[0]["key_id"], "key-codex-reserved");
|
||||||
|
assert_eq!(
|
||||||
|
keys[0]["scheduling_reason"] == "account_quota_exhausted",
|
||||||
|
exhausted
|
||||||
|
);
|
||||||
|
if exhausted {
|
||||||
|
assert_eq!(keys[0]["scheduling_status"], "blocked");
|
||||||
|
assert_eq!(keys[0]["scheduling_label"], "额度耗尽");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
|
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
|
||||||
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
|
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
|
||||||
|
|||||||
@@ -169,6 +169,18 @@ pub fn admin_pool_key_account_quota_exhausted(
|
|||||||
aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
|
aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn admin_pool_key_minimum_quota_reached(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
provider_type: &str,
|
||||||
|
provider_model_name: Option<&str>,
|
||||||
|
) -> bool {
|
||||||
|
aether_provider_pool::provider_pool_key_minimum_quota_reached(
|
||||||
|
key,
|
||||||
|
provider_type,
|
||||||
|
provider_model_name,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn admin_pool_key_quota_hard_blocked(
|
pub fn admin_pool_key_quota_hard_blocked(
|
||||||
key: &StoredProviderCatalogKey,
|
key: &StoredProviderCatalogKey,
|
||||||
provider_type: &str,
|
provider_type: &str,
|
||||||
|
|||||||
@@ -38,11 +38,11 @@ pub use providers::{
|
|||||||
XAI_BILLING_PATH, XAI_USER_PATH,
|
XAI_BILLING_PATH, XAI_USER_PATH,
|
||||||
};
|
};
|
||||||
pub use quota::{
|
pub use quota::{
|
||||||
provider_pool_key_account_quota_exhausted, provider_pool_key_model_quota_exhausted,
|
provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached,
|
||||||
provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked,
|
provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked,
|
||||||
provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot,
|
provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label,
|
||||||
provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at,
|
provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type,
|
||||||
provider_pool_quota_snapshot_updated_at,
|
provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at,
|
||||||
};
|
};
|
||||||
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
|
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
|
||||||
pub use service::ProviderPoolService;
|
pub use service::ProviderPoolService;
|
||||||
@@ -727,6 +727,275 @@ mod tests {
|
|||||||
assert!(unsupported.is_empty());
|
assert!(unsupported.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_respects_boundary_and_provider() {
|
||||||
|
for (used_percent, expected) in [
|
||||||
|
(98.9, false),
|
||||||
|
(98.99999, false),
|
||||||
|
(99.0, true),
|
||||||
|
(99.5, true),
|
||||||
|
(100.0, true),
|
||||||
|
] {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": { "primary_used_percent": used_percent }
|
||||||
|
})));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", None),
|
||||||
|
expected,
|
||||||
|
"used_percent={used_percent}"
|
||||||
|
);
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(&key, "kiro", None));
|
||||||
|
}
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&sample_key(None),
|
||||||
|
"codex",
|
||||||
|
None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_short_model_names_still_use_account_windows() {
|
||||||
|
for (used_percent, expected) in [(98.0, false), (99.0, true)] {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": { "primary_used_percent": used_percent }
|
||||||
|
})));
|
||||||
|
for model in ["o1", "o3", "", " "] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some(model)),
|
||||||
|
expected,
|
||||||
|
"model={model:?}, used_percent={used_percent}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_respects_windows_and_reset() {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.expect("system time should be after unix epoch")
|
||||||
|
.as_secs();
|
||||||
|
for (reset_at, expected) in [(now + 3600, true), (now - 60, false)] {
|
||||||
|
let mut key = sample_key(None);
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"updated_at": now - 600,
|
||||||
|
"windows": [
|
||||||
|
{ "code": "5h", "used_ratio": 0.2 },
|
||||||
|
{ "code": "weekly", "used_ratio": 0.99, "reset_at": reset_at }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
for model in [None, Some("gpt-5.4")] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", model),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_checks_either_window_and_relative_resets() {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.expect("system time should be after unix epoch")
|
||||||
|
.as_secs();
|
||||||
|
for prefix in ["primary", "secondary"] {
|
||||||
|
for (reset_seconds, expected) in [(3600, true), (60, false)] {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"updated_at": now - 600,
|
||||||
|
"allowed": true,
|
||||||
|
"limit_reached": false,
|
||||||
|
format!("{prefix}_used_percent"): 99.0,
|
||||||
|
format!("{prefix}_reset_after_seconds"): reset_seconds
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
for model in [None, Some("gpt-5.4")] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", model),
|
||||||
|
expected,
|
||||||
|
"prefix={prefix}, reset_seconds={reset_seconds}, model={model:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!provider_pool_key_account_quota_exhausted(&key, "codex"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_uses_newest_source_with_or_without_model() {
|
||||||
|
for (snapshot_at, metadata_at, metadata_wins) in [
|
||||||
|
(Some(200), Some(100), false),
|
||||||
|
(Some(100), Some(200), true),
|
||||||
|
(None, None, false),
|
||||||
|
(Some(100), None, false),
|
||||||
|
(None, Some(100), true),
|
||||||
|
] {
|
||||||
|
for snapshot_reached in [false, true] {
|
||||||
|
let mut key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"updated_at": metadata_at,
|
||||||
|
"primary_used_percent": if snapshot_reached { 20.0 } else { 99.0 }
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": snapshot_at,
|
||||||
|
"windows": [{
|
||||||
|
"code": "5h",
|
||||||
|
"used_ratio": if snapshot_reached { 0.99 } else { 0.2 },
|
||||||
|
"is_exhausted": false
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
for model in [None, Some("gpt-5.4")] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", model),
|
||||||
|
snapshot_reached != metadata_wins,
|
||||||
|
"snapshot_at={snapshot_at:?}, metadata_at={metadata_at:?}, model={model:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_isolates_model_buckets_and_checks_each_model_window() {
|
||||||
|
for explicit_model in [false, true] {
|
||||||
|
let mut key = sample_key(None);
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"windows": [
|
||||||
|
{ "code": "5h", "used_ratio": 0.2 },
|
||||||
|
{
|
||||||
|
"code": "spark_5h", "used_ratio": 0.99,
|
||||||
|
"model": if explicit_model { Some("spark") } else { None },
|
||||||
|
"is_exhausted": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"code": "spark_weekly", "used_ratio": 0.2,
|
||||||
|
"model": if explicit_model { Some("spark") } else { None },
|
||||||
|
"is_exhausted": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
assert!(provider_pool_key_minimum_quota_reached(
|
||||||
|
&key,
|
||||||
|
"codex",
|
||||||
|
Some("gpt-5.3-codex-spark")
|
||||||
|
));
|
||||||
|
for model in [None, Some("gpt-5.4")] {
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&key, "codex", model
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"),
|
||||||
|
Some(false)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (account_percent, spark_percent) in [(99.0, 20.0), (20.0, 99.0)] {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"primary_used_percent": account_percent,
|
||||||
|
"spark_primary_used_percent": spark_percent,
|
||||||
|
"spark_secondary_used_percent": 20.0
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
|
||||||
|
spark_percent == 99.0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.4")),
|
||||||
|
account_percent == 99.0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_keeps_model_bucket_when_account_metadata_is_newer() {
|
||||||
|
for (snapshot_ratio, account_percent) in [(0.2, 99.0), (0.99, 20.0)] {
|
||||||
|
let mut key = sample_key(Some(json!({
|
||||||
|
"codex": { "updated_at": 200, "primary_used_percent": account_percent }
|
||||||
|
})));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": 100,
|
||||||
|
"windows": [{ "code": "spark_5h", "used_ratio": snapshot_ratio }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
|
||||||
|
snapshot_ratio == 0.99
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.4")),
|
||||||
|
account_percent == 99.0
|
||||||
|
);
|
||||||
|
|
||||||
|
key.upstream_metadata.as_mut().unwrap()["codex"]["spark_primary_used_percent"] =
|
||||||
|
json!(account_percent);
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
|
||||||
|
account_percent == 99.0,
|
||||||
|
"the newer observation for the same model bucket must win"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_minimum_quota_supports_remaining_values_and_ignores_unknown_data() {
|
||||||
|
for (metrics, expected) in [
|
||||||
|
(json!({ "remaining_ratio": 0.01 }), true),
|
||||||
|
(json!({ "remaining_percent": "1" }), true),
|
||||||
|
(json!({ "remaining": 1, "limit": 100 }), true),
|
||||||
|
(json!({ "used_percent": "99" }), true),
|
||||||
|
(json!({ "used_ratio": null }), false),
|
||||||
|
(json!({ "used_ratio": "NaN" }), false),
|
||||||
|
(json!({ "remaining": 0, "limit": 0 }), false),
|
||||||
|
(json!({ "remaining_ratio": 0.01001 }), false),
|
||||||
|
] {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": { "quota_by_model": { "spark": metrics } }
|
||||||
|
})));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
|
||||||
|
expected,
|
||||||
|
"metrics={metrics}"
|
||||||
|
);
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&key, "codex", None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let disabled = sample_key(Some(json!({
|
||||||
|
"codex": { "primary_used_percent": 99.0, "primary_window_minutes": 0 }
|
||||||
|
})));
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&disabled, "codex", None
|
||||||
|
));
|
||||||
|
|
||||||
|
let mut mismatched = sample_key(None);
|
||||||
|
mismatched.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "kiro",
|
||||||
|
"windows": [{ "code": "weekly", "used_ratio": 0.99 }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&mismatched,
|
||||||
|
"codex",
|
||||||
|
None
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn provider_quota_exhaustion_is_adapter_owned() {
|
fn provider_quota_exhaustion_is_adapter_owned() {
|
||||||
assert!(provider_pool_key_account_quota_exhausted(
|
assert!(provider_pool_key_account_quota_exhausted(
|
||||||
|
|||||||
@@ -78,8 +78,17 @@ pub(crate) fn provider_pool_model_quota_exhausted(
|
|||||||
provider_type: &str,
|
provider_type: &str,
|
||||||
provider_model_name: &str,
|
provider_model_name: &str,
|
||||||
) -> Option<bool> {
|
) -> Option<bool> {
|
||||||
let requested = provider_pool_identifier_tokens(provider_model_name);
|
provider_pool_quota_reaches_reserve(key, provider_type, Some(provider_model_name), 0.0)
|
||||||
if requested.is_empty() {
|
}
|
||||||
|
|
||||||
|
fn provider_pool_quota_reaches_reserve(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
provider_type: &str,
|
||||||
|
provider_model_name: Option<&str>,
|
||||||
|
reserve_ratio: f64,
|
||||||
|
) -> Option<bool> {
|
||||||
|
let requested = provider_model_name.map(provider_pool_identifier_tokens);
|
||||||
|
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,8 +101,38 @@ pub(crate) fn provider_pool_model_quota_exhausted(
|
|||||||
provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type),
|
provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type),
|
||||||
];
|
];
|
||||||
let mut resolved = None::<(Option<u64>, bool)>;
|
let mut resolved = None::<(Option<u64>, bool)>;
|
||||||
|
let mut resolved_specific_bucket = false;
|
||||||
for source in sources.into_iter().flatten() {
|
for source in sources.into_iter().flatten() {
|
||||||
let windows = provider_pool_collect_quota_windows(source);
|
let mut windows = provider_pool_collect_quota_windows(source);
|
||||||
|
if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") {
|
||||||
|
// Raw refresh metadata can be newer than the materialized windows.
|
||||||
|
// Include all four legacy slots before selecting the model bucket.
|
||||||
|
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
|
||||||
|
let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
|
||||||
|
== Some(0.0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut window = Map::from_iter([
|
||||||
|
("code".to_string(), json!(prefix)),
|
||||||
|
("used_percent".to_string(), used_percent.clone()),
|
||||||
|
]);
|
||||||
|
for field in [
|
||||||
|
"reset_at",
|
||||||
|
"next_reset_at",
|
||||||
|
"reset_seconds",
|
||||||
|
"reset_after_seconds",
|
||||||
|
] {
|
||||||
|
if let Some(value) = source.get(&format!("{prefix}_{field}")) {
|
||||||
|
window.insert(field.to_string(), value.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
windows.push(window);
|
||||||
|
}
|
||||||
|
}
|
||||||
if windows.is_empty() {
|
if windows.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -102,19 +141,28 @@ pub(crate) fn provider_pool_model_quota_exhausted(
|
|||||||
let model_matches = windows
|
let model_matches = windows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|window| {
|
.filter(|window| {
|
||||||
provider_pool_window_explicitly_matches_model(window, provider_model_name)
|
provider_model_name.is_some_and(|model| {
|
||||||
|
provider_pool_window_explicitly_matches_model(window, model)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
if !model_matches.is_empty() {
|
if !model_matches.is_empty() {
|
||||||
let exhausted =
|
let exhausted = if reserve_ratio > 0.0 {
|
||||||
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at);
|
// Reserving quota protects every applicable rate-limit window,
|
||||||
|
// including short and weekly limits for an explicit model.
|
||||||
|
provider_pool_any_window_exhausted(model_matches, observed_at, reserve_ratio)
|
||||||
|
} else {
|
||||||
|
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
|
||||||
|
};
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
observed_at,
|
observed_at,
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
resolved = Some((observed_at, exhausted));
|
resolved = Some((observed_at, exhausted));
|
||||||
|
resolved_specific_bucket = true;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -125,21 +173,34 @@ pub(crate) fn provider_pool_model_quota_exhausted(
|
|||||||
// windows isolated without baking in names such as "spark".
|
// windows isolated without baking in names such as "spark".
|
||||||
let family_matches = windows
|
let family_matches = windows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|window| provider_pool_window_family_matches_model(window, &requested))
|
.filter(|window| {
|
||||||
|
requested
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|tokens| provider_pool_window_family_matches_model(window, tokens))
|
||||||
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
if !family_matches.is_empty() {
|
if !family_matches.is_empty() {
|
||||||
let exhausted = provider_pool_any_window_exhausted(family_matches, observed_at);
|
let exhausted =
|
||||||
|
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
observed_at,
|
observed_at,
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
resolved = Some((observed_at, exhausted));
|
resolved = Some((observed_at, exhausted));
|
||||||
|
resolved_specific_bucket = true;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A newer account observation does not update an independent model
|
||||||
|
// bucket. Only compare freshness between applicable model sources.
|
||||||
|
if reserve_ratio > 0.0 && resolved_specific_bucket {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Account-scoped windows (for example the ordinary weekly and short
|
// Account-scoped windows (for example the ordinary weekly and short
|
||||||
// windows emitted by Codex) apply to every model that has no more
|
// windows emitted by Codex) apply to every model that has no more
|
||||||
// specific family. Restrict this fallback to well-known structural
|
// specific family. Restrict this fallback to well-known structural
|
||||||
@@ -150,7 +211,8 @@ pub(crate) fn provider_pool_model_quota_exhausted(
|
|||||||
.filter(|window| provider_pool_window_is_generic(window))
|
.filter(|window| provider_pool_window_is_generic(window))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
if !generic_matches.is_empty() {
|
if !generic_matches.is_empty() {
|
||||||
let exhausted = provider_pool_any_window_exhausted(generic_matches, observed_at);
|
let exhausted =
|
||||||
|
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio);
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
@@ -209,16 +271,69 @@ fn provider_pool_explicit_model_windows_exhausted(
|
|||||||
fn provider_pool_any_window_exhausted(
|
fn provider_pool_any_window_exhausted(
|
||||||
windows: Vec<&Map<String, Value>>,
|
windows: Vec<&Map<String, Value>>,
|
||||||
snapshot_observed_at: Option<u64>,
|
snapshot_observed_at: Option<u64>,
|
||||||
|
reserve_ratio: f64,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let now_unix_secs = provider_pool_current_unix_secs();
|
let now_unix_secs = provider_pool_current_unix_secs();
|
||||||
windows.iter().any(|window| {
|
windows.iter().any(|window| {
|
||||||
provider_pool_quota_window_is_exhausted(window)
|
provider_pool_quota_window_reaches_reserve(window, reserve_ratio)
|
||||||
&& !now_unix_secs.is_some_and(|now| {
|
&& !now_unix_secs.is_some_and(|now| {
|
||||||
provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now)
|
provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether an applicable Codex window has at most 1% remaining. Missing quota
|
||||||
|
/// data and windows whose reset has elapsed do not trigger this opt-in guard.
|
||||||
|
pub fn provider_pool_key_minimum_quota_reached(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
provider_type: &str,
|
||||||
|
provider_model_name: Option<&str>,
|
||||||
|
) -> bool {
|
||||||
|
if !provider_type.trim().eq_ignore_ascii_case("codex") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
provider_pool_quota_reaches_reserve(key, provider_type, provider_model_name, 0.01)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_pool_quota_window_reaches_reserve(
|
||||||
|
window: &Map<String, Value>,
|
||||||
|
reserve_ratio: f64,
|
||||||
|
) -> bool {
|
||||||
|
if provider_pool_quota_window_is_exhausted(window) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if reserve_ratio <= 0.0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let used_ratio = provider_pool_json_f64(window.get("used_ratio"))
|
||||||
|
.or_else(|| provider_pool_json_f64(window.get("usage_ratio")))
|
||||||
|
.or_else(|| provider_pool_json_f64(window.get("used_percent")).map(|value| value / 100.0))
|
||||||
|
.or_else(|| {
|
||||||
|
provider_pool_json_f64(window.get("remaining_ratio"))
|
||||||
|
.or_else(|| provider_pool_json_f64(window.get("remaining_fraction")))
|
||||||
|
.map(|value| 1.0 - value)
|
||||||
|
})
|
||||||
|
.or_else(|| {
|
||||||
|
provider_pool_json_f64(window.get("remaining_percent")).map(|value| 1.0 - value / 100.0)
|
||||||
|
})
|
||||||
|
.or_else(|| {
|
||||||
|
let remaining = provider_pool_json_f64(
|
||||||
|
window
|
||||||
|
.get("remaining")
|
||||||
|
.or_else(|| window.get("remaining_value")),
|
||||||
|
)?;
|
||||||
|
let limit = provider_pool_json_f64(
|
||||||
|
window
|
||||||
|
.get("limit")
|
||||||
|
.or_else(|| window.get("limit_value"))
|
||||||
|
.or_else(|| window.get("total")),
|
||||||
|
)?;
|
||||||
|
(limit > 0.0).then_some(1.0 - remaining / limit)
|
||||||
|
});
|
||||||
|
used_ratio.is_some_and(|used_ratio| used_ratio >= 1.0 - reserve_ratio)
|
||||||
|
}
|
||||||
|
|
||||||
fn provider_pool_window_is_generic(window: &Map<String, Value>) -> bool {
|
fn provider_pool_window_is_generic(window: &Map<String, Value>) -> bool {
|
||||||
if provider_pool_window_has_explicit_model(window)
|
if provider_pool_window_has_explicit_model(window)
|
||||||
|| window
|
|| window
|
||||||
|
|||||||
@@ -822,6 +822,8 @@ export interface PoolAdvancedConfig {
|
|||||||
sticky_session_ttl_seconds?: number | null
|
sticky_session_ttl_seconds?: number | null
|
||||||
load_threshold_percent?: number | null
|
load_threshold_percent?: number | null
|
||||||
skip_exhausted_accounts?: boolean | null
|
skip_exhausted_accounts?: boolean | null
|
||||||
|
// Codex only: treat remaining quota <= 1% as exhausted (default false).
|
||||||
|
reserve_minimum_quota?: boolean
|
||||||
// 旧字段(兼容读取)
|
// 旧字段(兼容读取)
|
||||||
lru_enabled?: boolean
|
lru_enabled?: boolean
|
||||||
scheduling_mode?: 'lru' | 'multi_score' | null
|
scheduling_mode?: 'lru' | 'multi_score' | null
|
||||||
|
|||||||
@@ -72,6 +72,33 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-if="isCodex"
|
||||||
|
class="flex items-start justify-between gap-4 rounded-2xl border border-border/60 bg-card/70 p-4"
|
||||||
|
>
|
||||||
|
<div class="space-y-1">
|
||||||
|
<label
|
||||||
|
for="pool-reserve-minimum-quota"
|
||||||
|
class="text-sm font-medium"
|
||||||
|
>
|
||||||
|
保留最低额度
|
||||||
|
</label>
|
||||||
|
<p
|
||||||
|
id="pool-reserve-minimum-quota-description"
|
||||||
|
class="text-xs leading-5 text-muted-foreground"
|
||||||
|
>
|
||||||
|
账号剩余额度不高于 1% 时提前标记为额度耗尽并停止调度,待额度恢复后再使用。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<Switch
|
||||||
|
id="pool-reserve-minimum-quota"
|
||||||
|
v-model="reserveMinimumQuota"
|
||||||
|
aria-describedby="pool-reserve-minimum-quota-description"
|
||||||
|
:disabled="loading"
|
||||||
|
class="mt-0.5 shrink-0"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Section 2: 策略调度 (非互斥, 可叠加组合 + 拖拽排序) -->
|
<!-- Section 2: 策略调度 (非互斥, 可叠加组合 + 拖拽排序) -->
|
||||||
<div class="space-y-4 rounded-2xl border border-border/60 bg-card/70 p-4">
|
<div class="space-y-4 rounded-2xl border border-border/60 bg-card/70 p-4">
|
||||||
<div class="space-y-1">
|
<div class="space-y-1">
|
||||||
@@ -437,6 +464,8 @@ const FALLBACK_PRESET_DEFS: PoolPresetMeta[] = [
|
|||||||
|
|
||||||
const { success, error: showError } = useToast()
|
const { success, error: showError } = useToast()
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
|
const reserveMinimumQuota = ref(false)
|
||||||
|
const isCodex = computed(() => normalizeProviderType(props.providerType) === 'codex')
|
||||||
let dialogRevision = 0
|
let dialogRevision = 0
|
||||||
const presetDefs = ref<PoolPresetMeta[]>([])
|
const presetDefs = ref<PoolPresetMeta[]>([])
|
||||||
const presetDefsLoaded = ref(false)
|
const presetDefsLoaded = ref(false)
|
||||||
@@ -850,6 +879,7 @@ watch([() => props.modelValue, () => props.providerId], async ([open]) => {
|
|||||||
const revision = ++dialogRevision
|
const revision = ++dialogRevision
|
||||||
loading.value = false
|
loading.value = false
|
||||||
if (!open) return
|
if (!open) return
|
||||||
|
reserveMinimumQuota.value = props.currentConfig?.reserve_minimum_quota === true
|
||||||
await ensurePresetDefsLoaded()
|
await ensurePresetDefsLoaded()
|
||||||
if (!props.modelValue || dialogRevision !== revision) return
|
if (!props.modelValue || dialogRevision !== revision) return
|
||||||
presetList.value = normalizeMutexSelection(loadFromConfig(props.currentConfig))
|
presetList.value = normalizeMutexSelection(loadFromConfig(props.currentConfig))
|
||||||
@@ -883,6 +913,7 @@ async function handleSave() {
|
|||||||
const latestAdvanced = latestProvider.pool_advanced
|
const latestAdvanced = latestProvider.pool_advanced
|
||||||
const mergedAdvanced = mergePoolAdvancedPatch(latestAdvanced, {
|
const mergedAdvanced = mergePoolAdvancedPatch(latestAdvanced, {
|
||||||
scheduling_presets: schedulingPresets,
|
scheduling_presets: schedulingPresets,
|
||||||
|
...(isCodex.value ? { reserve_minimum_quota: reserveMinimumQuota.value } : {}),
|
||||||
})
|
})
|
||||||
const payload: Parameters<typeof updateProvider>[1] = {
|
const payload: Parameters<typeof updateProvider>[1] = {
|
||||||
pool_advanced: mergedAdvanced as PoolAdvancedConfig,
|
pool_advanced: mergedAdvanced as PoolAdvancedConfig,
|
||||||
|
|||||||
+166
@@ -0,0 +1,166 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
import { createApp, defineComponent, h, nextTick, ref, type App } from 'vue'
|
||||||
|
|
||||||
|
import type { PoolAdvancedConfig } from '@/api/endpoints/types/provider'
|
||||||
|
import PoolSchedulingDialog from '../PoolSchedulingDialog.vue'
|
||||||
|
|
||||||
|
const endpointMocks = vi.hoisted(() => ({
|
||||||
|
getPoolSchedulingPresets: vi.fn(),
|
||||||
|
getProvider: vi.fn(),
|
||||||
|
updateProvider: vi.fn(),
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.mock('@/api/endpoints', () => ({
|
||||||
|
getProvider: endpointMocks.getProvider,
|
||||||
|
updateProvider: endpointMocks.updateProvider,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.mock('@/api/endpoints/pool', () => ({
|
||||||
|
getPoolSchedulingPresets: endpointMocks.getPoolSchedulingPresets,
|
||||||
|
}))
|
||||||
|
|
||||||
|
vi.mock('@/composables/useToast', () => ({
|
||||||
|
useToast: () => ({ success: vi.fn(), error: vi.fn() }),
|
||||||
|
}))
|
||||||
|
|
||||||
|
const mountedApps: Array<{ app: App, root: HTMLElement }> = []
|
||||||
|
|
||||||
|
async function settle(): Promise<void> {
|
||||||
|
for (let index = 0; index < 4; index += 1) {
|
||||||
|
await Promise.resolve()
|
||||||
|
await nextTick()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function mountDialog(providerType = 'codex', currentConfig: PoolAdvancedConfig | null = null) {
|
||||||
|
const root = document.createElement('div')
|
||||||
|
document.body.appendChild(root)
|
||||||
|
const open = ref(false)
|
||||||
|
const TestHost = defineComponent({
|
||||||
|
setup() {
|
||||||
|
void nextTick(() => { open.value = true })
|
||||||
|
return () => h(PoolSchedulingDialog, {
|
||||||
|
modelValue: open.value,
|
||||||
|
providerId: 'provider-1',
|
||||||
|
providerType,
|
||||||
|
currentConfig,
|
||||||
|
'onUpdate:modelValue': (value: boolean) => { open.value = value },
|
||||||
|
})
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const app = createApp(TestHost)
|
||||||
|
app.mount(root)
|
||||||
|
mountedApps.push({ app, root })
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
function quotaSwitch() {
|
||||||
|
return document.body.querySelector<HTMLButtonElement>('#pool-reserve-minimum-quota')
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveDialog() {
|
||||||
|
const saveButton = [...document.body.querySelectorAll<HTMLButtonElement>('button')]
|
||||||
|
.find(button => button.textContent?.trim() === '保存')
|
||||||
|
expect(saveButton).toBeDefined()
|
||||||
|
saveButton?.click()
|
||||||
|
await settle()
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
endpointMocks.getPoolSchedulingPresets.mockReset()
|
||||||
|
endpointMocks.getProvider.mockReset()
|
||||||
|
endpointMocks.updateProvider.mockReset()
|
||||||
|
endpointMocks.getPoolSchedulingPresets.mockResolvedValue([{
|
||||||
|
name: 'lru',
|
||||||
|
label: 'LRU 轮转',
|
||||||
|
description: '最久未使用的 Key 优先',
|
||||||
|
providers: [],
|
||||||
|
default_enabled: true,
|
||||||
|
modes: null,
|
||||||
|
default_mode: null,
|
||||||
|
mutex_group: 'distribution_mode',
|
||||||
|
}])
|
||||||
|
endpointMocks.getProvider.mockResolvedValue({ id: 'provider-1', pool_advanced: {} })
|
||||||
|
endpointMocks.updateProvider.mockResolvedValue({ id: 'provider-1' })
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const { app, root } of mountedApps.splice(0)) {
|
||||||
|
app.unmount()
|
||||||
|
root.remove()
|
||||||
|
}
|
||||||
|
document.body.innerHTML = ''
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('PoolSchedulingDialog minimum quota reserve', () => {
|
||||||
|
it('defaults to disabled and saves enabling it while preserving the latest settings', async () => {
|
||||||
|
endpointMocks.getProvider.mockResolvedValue({
|
||||||
|
id: 'provider-1',
|
||||||
|
pool_advanced: { rate_limit_cooldown_seconds: 900, skip_exhausted_accounts: false },
|
||||||
|
})
|
||||||
|
mountDialog('codex', { rate_limit_cooldown_seconds: 300 })
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('false')
|
||||||
|
expect(document.body.textContent).toContain('剩余额度不高于 1%')
|
||||||
|
quotaSwitch()?.click()
|
||||||
|
await nextTick()
|
||||||
|
await saveDialog()
|
||||||
|
|
||||||
|
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
|
||||||
|
pool_advanced: {
|
||||||
|
rate_limit_cooldown_seconds: 900,
|
||||||
|
skip_exhausted_accounts: false,
|
||||||
|
reserve_minimum_quota: true,
|
||||||
|
scheduling_presets: [{ preset: 'lru', enabled: true }],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loads an enabled reserve and saves disabling it', async () => {
|
||||||
|
mountDialog('codex', { reserve_minimum_quota: true })
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('true')
|
||||||
|
quotaSwitch()?.click()
|
||||||
|
await nextTick()
|
||||||
|
await saveDialog()
|
||||||
|
|
||||||
|
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
|
||||||
|
pool_advanced: {
|
||||||
|
reserve_minimum_quota: false,
|
||||||
|
scheduling_presets: [{ preset: 'lru', enabled: true }],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('discards unsaved reserve changes when the dialog is reopened', async () => {
|
||||||
|
const open = mountDialog()
|
||||||
|
await settle()
|
||||||
|
quotaSwitch()?.click()
|
||||||
|
await nextTick()
|
||||||
|
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('true')
|
||||||
|
|
||||||
|
open.value = false
|
||||||
|
await settle()
|
||||||
|
open.value = true
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('false')
|
||||||
|
expect(endpointMocks.updateProvider).not.toHaveBeenCalled()
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each(['openai', 'kiro'])('does not expose or add the reserve setting for %s', async (providerType) => {
|
||||||
|
mountDialog(providerType)
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(quotaSwitch()).toBeNull()
|
||||||
|
await saveDialog()
|
||||||
|
|
||||||
|
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
|
||||||
|
pool_advanced: {
|
||||||
|
scheduling_presets: [{ preset: 'lru', enabled: true }],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1662,6 +1662,8 @@ export const legacyUiEnglishMessages: Record<string, string> = {
|
|||||||
'依据窗口成本/Token 用量,缺失时回退配额使用率': 'Use window cost or token usage, falling back to quota utilization if unavailable',
|
'依据窗口成本/Token 用量,缺失时回退配额使用率': 'Use window cost or token usage, falling back to quota utilization if unavailable',
|
||||||
'调度策略元数据加载失败,请重试': 'Failed to load scheduling policy metadata. Try again.',
|
'调度策略元数据加载失败,请重试': 'Failed to load scheduling policy metadata. Try again.',
|
||||||
'号池调度已保存': 'Pool scheduling saved',
|
'号池调度已保存': 'Pool scheduling saved',
|
||||||
|
'保留最低额度': 'Reserve minimum quota',
|
||||||
|
'账号剩余额度不高于 1% 时提前标记为额度耗尽并停止调度,待额度恢复后再使用。': 'Mark accounts as exhausted and stop scheduling when remaining quota is 1% or less, until quota recovers.',
|
||||||
'自适应热池': 'Adaptive warm pool',
|
'自适应热池': 'Adaptive warm pool',
|
||||||
'自动维护热池,缺口时异步补位。': 'Maintain a warm pool and replenish missing capacity asynchronously.',
|
'自动维护热池,缺口时异步补位。': 'Maintain a warm pool and replenish missing capacity asynchronously.',
|
||||||
'账号自检': 'Account self-check',
|
'账号自检': 'Account self-check',
|
||||||
|
|||||||
Reference in New Issue
Block a user