feat(adaptive): 完善自适应 RPM 学习并将 Pool 调度状态与健康分解耦

- orchestration 新增 AdaptiveSuccess 效果,在成功回报路径上根据利用率窗口扩张 learned_rpm_limit
- 429 路径改用 429_observation/adjustment 记录以及基于历史的置信度评估,新增 last_rpm_peak 边界字段
- Pool 调度状态不再因 health_score 低或熔断而降级/拦截,前端同步移除相关按钮与文案兜底
- 新增前端 poolTrace 工具(附测试)承接原 HorizontalRequestTimeline 内的候选合并逻辑
This commit is contained in:
fawney19
2026-04-20 00:27:48 +08:00
parent 1f74e660de
commit c1b9d94c84
16 changed files with 1776 additions and 405 deletions

View File

@@ -498,6 +498,13 @@ import { log } from '@/utils/logger'
import { parseApiError } from '@/utils/errorParser'
import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { resolveTimelineFinalStatus } from '../utils/status'
import {
buildPoolAttemptCandidatesFromAudit,
extractPoolGroupId,
isPoolAttemptedCandidate,
makeAttemptKey,
TIMELINE_STATUS,
} from '../utils/poolTrace'
// 节点组类型
interface NodeGroup {
@@ -691,18 +698,6 @@ const proxyTimingBreakdown = (proxy: Record<string, unknown>): string => {
return parts.join(' / ')
}
const TIMELINE_STATUS: CandidateRecord['status'][] = [
'success',
'failed',
'skipped',
'cancelled',
'pending',
'streaming',
'available',
'unused',
'stream_interrupted',
]
const STATUS_PRIORITY: Record<string, number> = {
available: 0,
unused: 0,
@@ -715,47 +710,6 @@ const STATUS_PRIORITY: Record<string, number> = {
success: 4,
}
const toInt = (value: unknown, defaultValue = 0): number => {
const num = Number(value)
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
}
const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
return `${candidateIndex}:${retryIndex}`
}
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
'available',
'unused',
'skipped',
])
const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
// pending 只有开始执行后才算真正进入号池内部尝试
if (candidate.status === 'pending' && !candidate.started_at) return false
return true
}
const normalizeTimelineStatus = (value: unknown): CandidateRecord['status'] => {
if (typeof value !== 'string') return 'failed'
const normalized = value.trim().toLowerCase()
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
return normalized as CandidateRecord['status']
}
// 兜底:内部调度轨迹里非标准状态统一按失败展示
return 'failed'
}
const extractPoolGroupId = (candidate: CandidateRecord): string | null => {
const extra = candidate.extra_data
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
const value = (extra as Record<string, unknown>).pool_group_id
if (typeof value !== 'string') return null
const text = value.trim()
return text || null
}
// 候选时间线(按实际执行顺序排序)
const rawTimeline = computed<CandidateRecord[]>(() => {
if (!trace.value) return []
@@ -794,97 +748,11 @@ const poolAttemptCandidates = computed<CandidateRecord[]>(() => {
// 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。
const audit = schedulingAudit.value
if (!audit) return []
const attempts = audit.attempts
if (!Array.isArray(attempts) || attempts.length === 0) return []
const providerNameById = new Map<string, string>()
for (const candidate of rawTimeline.value) {
const providerId = String(candidate.provider_id || '').trim()
const providerName = String(candidate.provider_name || '').trim()
if (!providerId || !providerName) continue
if (!providerNameById.has(providerId)) {
providerNameById.set(providerId, providerName)
}
}
const providerTypeLikeNames = new Set<string>([
'codex',
'kiro',
'antigravity',
'claude_code',
'claude code',
'gemini_cli',
'gemini cli',
'oauth',
'api_key',
'api key',
])
const traceMap = new Map<string, CandidateRecord>()
for (const candidate of rawTimeline.value) {
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
}
return attempts
.map((item, index) => {
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
const raw = item as Record<string, unknown>
const candidateIndex = toInt(raw.candidate_index, index)
const retryIndex = toInt(raw.retry_index, 0)
const key = makeAttemptKey(candidateIndex, retryIndex)
const fromTrace = traceMap.get(key)
const merged: CandidateRecord = fromTrace
? { ...fromTrace }
: {
id: `pool-${props.requestId}-${candidateIndex}-${retryIndex}-${index}`,
request_id: props.requestId,
candidate_index: candidateIndex,
retry_index: retryIndex,
provider_id: undefined,
provider_name: undefined,
endpoint_id: undefined,
key_id: undefined,
key_name: undefined,
status: 'failed',
is_cached: false,
created_at: new Date(0).toISOString(),
}
merged.status = normalizeTimelineStatus(raw.status ?? merged.status)
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
if (finalPoolGroupId) {
merged.extra_data = {
...(merged.extra_data || {}),
pool_group_id: finalPoolGroupId,
}
}
const mergedProviderId = String(merged.provider_id || '').trim()
if (mergedProviderId) {
const inferredProviderName = providerNameById.get(mergedProviderId)
const currentProviderName = String(merged.provider_name || '').trim()
if (
inferredProviderName
&& (
!currentProviderName
|| providerTypeLikeNames.has(currentProviderName.toLowerCase())
)
) {
merged.provider_name = inferredProviderName
}
}
return merged
})
.filter((item): item is CandidateRecord => item !== null)
return buildPoolAttemptCandidatesFromAudit(
rawTimeline.value,
audit.attempts,
props.requestId,
)
})
const poolAttemptsByGroup = computed<Map<string, CandidateRecord[]>>(() => {

View File

@@ -0,0 +1,114 @@
import { describe, expect, it } from 'vitest'
import type { CandidateRecord } from '@/api/requestTrace'
import { buildPoolAttemptCandidatesFromAudit } from '@/features/usage/utils/poolTrace'
function buildCandidate(
overrides: Partial<CandidateRecord> = {},
): CandidateRecord {
return {
id: 'cand-1',
request_id: 'req-1',
candidate_index: 0,
retry_index: 0,
status: 'failed',
is_cached: false,
created_at: '1970-01-01T00:00:00.000Z',
...overrides,
}
}
describe('poolTrace', () => {
it('keeps only actually attempted pool nodes from scheduling audit fallback', () => {
const attempts = buildPoolAttemptCandidatesFromAudit([], [
{
candidate_index: 0,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-success',
key_name: 'Success Key',
status: 'success',
pool_group_id: 'provider-1',
},
{
candidate_index: 1,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-skipped',
key_name: 'Skipped Key',
status: 'skipped',
pool_group_id: 'provider-1',
},
{
candidate_index: 2,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-available',
key_name: 'Available Key',
status: 'available',
pool_group_id: 'provider-1',
},
{
candidate_index: 3,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-unknown',
key_name: 'Unknown Key',
status: 'selected',
pool_group_id: 'provider-1',
},
], 'req-1')
expect(attempts).toHaveLength(1)
expect(attempts[0].key_id).toBe('key-success')
expect(attempts[0].status).toBe('success')
})
it('preserves real trace attempts even when audit status is non-standard', () => {
const rawTimeline = [
buildCandidate({
id: 'cand-trace-1',
candidate_index: 4,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-trace',
key_name: 'Trace Key',
status: 'failed',
started_at: '2026-04-19T12:00:00.000Z',
}),
]
const attempts = buildPoolAttemptCandidatesFromAudit(rawTimeline, [
{
candidate_index: 4,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'oauth',
key_id: 'key-trace',
key_name: 'Trace Key',
status: 'selected',
pool_group_id: 'provider-1',
},
{
candidate_index: 5,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'oauth',
key_id: 'key-ghost',
key_name: 'Ghost Key',
status: 'selected',
pool_group_id: 'provider-1',
},
], 'req-1')
expect(attempts).toHaveLength(1)
expect(attempts[0].id).toBe('cand-trace-1')
expect(attempts[0].status).toBe('failed')
expect(attempts[0].provider_name).toBe('Codex反代')
})
})

View File

@@ -0,0 +1,160 @@
import type { CandidateRecord } from '@/api/requestTrace'
export const TIMELINE_STATUS: CandidateRecord['status'][] = [
'success',
'failed',
'skipped',
'cancelled',
'pending',
'streaming',
'available',
'unused',
'stream_interrupted',
]
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
'available',
'unused',
'skipped',
])
const PROVIDER_TYPE_LIKE_NAMES = new Set<string>([
'codex',
'kiro',
'antigravity',
'claude_code',
'claude code',
'gemini_cli',
'gemini cli',
'oauth',
'api_key',
'api key',
])
const toInt = (value: unknown, defaultValue = 0): number => {
const num = Number(value)
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
}
export const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
return `${candidateIndex}:${retryIndex}`
}
export const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
if (candidate.status === 'pending' && !candidate.started_at) return false
return true
}
export const parseTimelineStatus = (value: unknown): CandidateRecord['status'] | null => {
if (typeof value !== 'string') return null
const normalized = value.trim().toLowerCase()
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
return normalized as CandidateRecord['status']
}
return null
}
export const extractPoolGroupId = (
candidate: Pick<CandidateRecord, 'extra_data'>,
): string | null => {
const extra = candidate.extra_data
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
const value = (extra as Record<string, unknown>).pool_group_id
if (typeof value !== 'string') return null
const text = value.trim()
return text || null
}
export function buildPoolAttemptCandidatesFromAudit(
rawTimeline: CandidateRecord[],
attempts: unknown,
requestId?: string | null,
): CandidateRecord[] {
if (!Array.isArray(attempts) || attempts.length === 0) return []
const providerNameById = new Map<string, string>()
for (const candidate of rawTimeline) {
const providerId = String(candidate.provider_id || '').trim()
const providerName = String(candidate.provider_name || '').trim()
if (!providerId || !providerName) continue
if (!providerNameById.has(providerId)) {
providerNameById.set(providerId, providerName)
}
}
const traceMap = new Map<string, CandidateRecord>()
for (const candidate of rawTimeline) {
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
}
return attempts
.map((item, index) => {
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
const raw = item as Record<string, unknown>
const candidateIndex = toInt(raw.candidate_index, index)
const retryIndex = toInt(raw.retry_index, 0)
const key = makeAttemptKey(candidateIndex, retryIndex)
const fromTrace = traceMap.get(key)
const parsedStatus = parseTimelineStatus(raw.status)
if (!fromTrace && parsedStatus === null) {
return null
}
const merged: CandidateRecord = fromTrace
? { ...fromTrace }
: {
id: `pool-${requestId || 'unknown'}-${candidateIndex}-${retryIndex}-${index}`,
request_id: requestId || '',
candidate_index: candidateIndex,
retry_index: retryIndex,
provider_id: undefined,
provider_name: undefined,
endpoint_id: undefined,
key_id: undefined,
key_name: undefined,
status: 'failed',
is_cached: false,
created_at: new Date(0).toISOString(),
}
if (parsedStatus !== null) {
merged.status = parsedStatus
}
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
if (finalPoolGroupId) {
merged.extra_data = {
...(merged.extra_data || {}),
pool_group_id: finalPoolGroupId,
}
}
const mergedProviderId = String(merged.provider_id || '').trim()
if (mergedProviderId) {
const inferredProviderName = providerNameById.get(mergedProviderId)
const currentProviderName = String(merged.provider_name || '').trim()
if (
inferredProviderName
&& (
!currentProviderName
|| PROVIDER_TYPE_LIKE_NAMES.has(currentProviderName.toLowerCase())
)
) {
merged.provider_name = inferredProviderName
}
}
return isPoolAttemptedCandidate(merged) ? merged : null
})
.filter((item): item is CandidateRecord => item !== null)
}

View File

@@ -604,20 +604,6 @@
>
<RefreshCw class="w-3.5 h-3.5" />
</Button>
<Button
v-if="key.circuit_breaker_open || (key.health_score ?? 1) < 0.5"
variant="ghost"
size="icon"
class="h-7 w-7 text-green-600"
:disabled="recoveringHealthKeyId === key.key_id"
title="刷新健康状态"
@click="handleRecoverKey(key)"
>
<RefreshCw
class="w-3.5 h-3.5"
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
/>
</Button>
<Button
variant="ghost"
size="icon"
@@ -896,20 +882,6 @@
>
<RefreshCw class="w-3.5 h-3.5" />
</Button>
<Button
v-else-if="actionId === 'recover_health'"
variant="ghost"
size="icon"
class="h-7 w-7 shrink-0 text-green-600"
:disabled="recoveringHealthKeyId === key.key_id"
title="刷新健康状态"
@click="handleRecoverKey(key)"
>
<RefreshCw
class="w-3.5 h-3.5"
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
/>
</Button>
<Button
v-else-if="actionId === 'permissions'"
variant="ghost"
@@ -1167,7 +1139,6 @@ import {
refreshProviderQuota,
} from '@/api/endpoints/keys'
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
import { recoverKeyHealth } from '@/api/endpoints/health'
import type {
PoolOverviewItem,
PoolKeyDetail,
@@ -1565,7 +1536,6 @@ const currentPage = ref(restoredViewState.page)
const pageSize = ref(restoredViewState.pageSize)
const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60
const refreshingOAuthKeyId = ref<string | null>(null)
const recoveringHealthKeyId = ref<string | null>(null)
const savingProxyKeyId = ref<string | null>(null)
const proxyDesktopPopoverOpenKeyId = ref<string | null>(null)
const proxyMobilePopoverOpenKeyId = ref<string | null>(null)
@@ -2075,20 +2045,6 @@ async function clearKeyProxy(key: PoolKeyDetail) {
}
}
async function handleRecoverKey(key: PoolKeyDetail) {
if (recoveringHealthKeyId.value) return
recoveringHealthKeyId.value = key.key_id
try {
const result = await recoverKeyHealth(key.key_id)
success(result.message || 'Key 已恢复')
await loadKeys()
} catch (err) {
showError(parseApiError(err, 'Key恢复失败'))
} finally {
recoveringHealthKeyId.value = null
}
}
async function handleDeleteKey(key: PoolKeyDetail) {
const confirmed = await confirm({
title: '删除账号',
@@ -2399,19 +2355,54 @@ function formatCooldownReason(reason: string): string {
type PoolStatusVariant = 'default' | 'secondary' | 'destructive' | 'outline' | 'success' | 'warning' | 'dark'
function isHealthDerivedSchedulingReason(reason: string | null | undefined): boolean {
const normalized = String(reason || '').trim().toLowerCase()
return normalized === 'health_low'
|| normalized === 'health_degraded'
|| normalized === 'health'
|| normalized === 'circuit_open'
|| normalized === 'circuit_breaker'
}
function isHealthDerivedSchedulingLabel(label: string | null | undefined): boolean {
const normalized = String(label || '').trim()
return normalized === '健康低'
|| normalized === '健康度较低'
|| normalized === '降级'
|| normalized === '熔断'
|| normalized === '熔断中'
}
function getVisibleSchedulingReason(key: PoolKeyDetail): string | null {
const reason = String(key.scheduling_reason || '').trim()
if (!reason || isHealthDerivedSchedulingReason(reason)) return null
return reason
}
function getVisibleSchedulingReasons(key: PoolKeyDetail) {
return (key.scheduling_reasons ?? []).filter((item) => {
const source = String(item.source || '').trim().toLowerCase()
return source !== 'health'
&& !isHealthDerivedSchedulingReason(item.code)
&& !isHealthDerivedSchedulingLabel(item.label)
})
}
function getSchedulingStatus(key: PoolKeyDetail): 'available' | 'degraded' | 'blocked' {
if (getAccountAlertLabel(key)) return 'blocked'
const status = key.scheduling_status
if (status === 'available' || status === 'degraded' || status === 'blocked') {
if (
(status === 'available' || status === 'degraded' || status === 'blocked')
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
&& !isHealthDerivedSchedulingLabel(key.scheduling_label)
) {
return status
}
if (!key.is_active) return 'blocked'
if (key.cooldown_reason) return 'blocked'
if (key.circuit_breaker_open) return 'blocked'
if (key.cooldown_reason) return 'degraded'
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return 'blocked'
if ((key.health_score ?? 1) < 0.8) return 'degraded'
return 'available'
}
@@ -2420,29 +2411,31 @@ function getSchedulingBadgeLabel(key: PoolKeyDetail): string {
if (accountAlert) return accountAlert
const rawLabel = String(key.scheduling_label || '').trim()
if (rawLabel) {
if (
rawLabel
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
&& !isHealthDerivedSchedulingLabel(rawLabel)
) {
if (rawLabel === '禁用' || rawLabel === '停用') return '禁用'
return rawLabel
}
if (!key.is_active) return '禁用'
if (key.cooldown_reason) return '冷却'
if (key.circuit_breaker_open) return '熔断'
if (key.cooldown_reason) return '冷却'
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return '超限'
if ((key.health_score ?? 1) < 0.5) return '健康低'
if ((key.health_score ?? 1) < 0.8) return '降级'
return '可用'
}
function getSchedulingBadgeVariant(key: PoolKeyDetail): PoolStatusVariant {
if (getAccountAlertLabel(key)) return 'destructive'
const reason = key.scheduling_reason
if (reason === 'manual_disabled') return 'secondary'
if (reason === 'cooldown' || reason === 'circuit_open' || reason === 'cost_exhausted') return 'destructive'
const reason = getVisibleSchedulingReason(key)
if (reason === 'manual_disabled' || reason === 'inactive') return 'secondary'
if (reason === 'account_blocked' || reason === 'account_quota_exhausted' || reason === 'cost_exhausted') return 'destructive'
if (reason === 'cooldown') return 'warning'
if (reason === 'cost_soft' || reason === 'cost') return 'warning'
if (reason === 'health_low' || reason === 'health_degraded' || reason === 'health') return 'warning'
if (reason === 'available') return 'default'
if (!reason && !key.is_active) return 'secondary'
const status = getSchedulingStatus(key)
if (status === 'blocked') return 'destructive'
@@ -2454,7 +2447,7 @@ function getSchedulingTitle(key: PoolKeyDetail): string {
const accountAlertTitle = getAccountAlertTitle(key)
if (accountAlertTitle) return accountAlertTitle
const reasons = key.scheduling_reasons ?? []
const reasons = getVisibleSchedulingReasons(key)
if (reasons.length > 0) {
return reasons.map((item) => {
const ttl = item.ttl_seconds && item.ttl_seconds > 0 ? ` (${formatTTL(item.ttl_seconds)})` : ''
@@ -2518,7 +2511,6 @@ function getMobileActionIds(key: PoolKeyDetail): PoolMobileActionId[] {
canDownloadOrCopy: true,
canRefreshToken: canRefreshOAuthCredential(key),
canClearCooldown: Boolean(key.cooldown_reason),
canRecoverHealth: key.circuit_breaker_open || (key.health_score ?? 1) < 0.5,
hasProxy: true,
}).primary
}
@@ -2669,7 +2661,7 @@ function getQuotaProgressCountdown(item: QuotaProgressItem) {
function getQuotaProgressCountdownText(item: QuotaProgressItem): string {
const status = getQuotaProgressCountdown(item)
if (!status) return ''
return status.isExpired ? status.text : `${status.text} 后重置`
return status.isExpired ? '' : `${status.text} 后重置`
}
function formatCompactQuotaCountdownText(text: string): string {
@@ -2681,10 +2673,15 @@ function formatCompactQuotaCountdownText(text: string): string {
return normalized.replace(/\s+后重置$/, '')
}
function shouldHideQuotaProgressDetailText(text: string | null | undefined): boolean {
return (text ?? '').trim().includes('已重置')
}
function getQuotaProgressDisplayText(item: QuotaProgressItem): string {
const countdownText = getQuotaProgressCountdownText(item)
if (countdownText) return formatCompactQuotaCountdownText(countdownText)
return item.detail?.trim() || ''
const detail = item.detail?.trim() || ''
return shouldHideQuotaProgressDetailText(detail) ? '' : detail
}
function getQuotaFallbackText(key: PoolKeyDetail): string | null {