feat(usage): enrich audit metadata and detail views

This commit is contained in:
elky
2026-07-17 19:20:16 +08:00
parent 0be380243b
commit 664c063a06
79 changed files with 6968 additions and 1294 deletions
@@ -0,0 +1,45 @@
import { describe, expect, it } from 'vitest'
import { isCyberPolicyError } from '../cyberError'
describe('isCyberPolicyError', () => {
it('recognizes the provider cybersecurity refusal message', () => {
expect(isCyberPolicyError({
error: {
type: 'invalid_request',
message: 'This content was flagged for possible cybersecurity risk. To get authorized for security work, join the Trusted Access for Cyber program: https://chatgpt.com/cyber',
code: 400,
},
})).toBe(true)
})
it('recognizes an explicit cyber_policy code', () => {
expect(isCyberPolicyError({ error: { code: 'CYBER_POLICY' } })).toBe(true)
})
it('recognizes explicit Cyber Policy types and reasons', () => {
expect(isCyberPolicyError({ error: { type: 'cyber_policy' } })).toBe(true)
expect(isCyberPolicyError({ error: { type: 'CYBER' } })).toBe(true)
expect(isCyberPolicyError({ error: { reason: 'cyber-policy' } })).toBe(true)
expect(isCyberPolicyError({ error: { category: 'cyber_policy_violation' } })).toBe(true)
expect(isCyberPolicyError({ error: { type: 'cybersecurity-risk' } })).toBe(true)
})
it('recognizes structured Cyber classifiers inside a serialized error', () => {
expect(isCyberPolicyError('{"error":{"type":"cyber"}}')).toBe(true)
})
it('does not classify ordinary invalid requests as Cyber Policy failures', () => {
expect(isCyberPolicyError({
error: {
type: 'invalid_request',
message: 'The request payload is malformed',
code: 400,
},
})).toBe(false)
})
it('does not classify a generic use of the word cyber', () => {
expect(isCyberPolicyError('The cyber security report was generated successfully')).toBe(false)
})
})
@@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest'
import type { UsageRecord } from '../../types'
import {
mergeUsageRecordErrorMessage,
mergeUsageRecordFirstByteTimeMs,
mergeUsageRecordLifecycleSnapshot,
mergeUsageRecordResponseTiming,
syncUsageRecordStreamResolution,
} from '../recordSync'
@@ -75,3 +78,140 @@ describe('mergeUsageRecordFirstByteTimeMs', () => {
expect(mergeUsageRecordFirstByteTimeMs(-1, null)).toBeUndefined()
})
})
describe('mergeUsageRecordResponseTiming', () => {
it('keeps duration and update timestamp as one monotonic active snapshot', () => {
const existing = {
response_time_ms: 5500,
response_time_updated_at: '2026-07-17T12:00:06Z',
}
const stale = {
response_time_ms: 5000,
response_time_updated_at: '2026-07-17T12:00:07Z',
}
expect(mergeUsageRecordResponseTiming(existing, stale)).toBe(existing)
})
it('accepts a live snapshot whose projected elapsed time has advanced', () => {
const existing = {
response_time_ms: 5500,
response_time_updated_at: '2026-07-17T12:00:06Z',
}
const advanced = {
response_time_ms: 7000,
response_time_updated_at: '2026-07-17T12:00:07Z',
}
expect(mergeUsageRecordResponseTiming(existing, advanced)).toBe(advanced)
})
it('does not combine an unanchored detail estimate with an existing anchor', () => {
const existing = {
response_time_ms: 5500,
response_time_updated_at: '2026-07-17T12:00:06Z',
}
const detailEstimate = {
response_time_ms: 6000,
response_time_updated_at: null,
}
expect(mergeUsageRecordResponseTiming(existing, detailEstimate)).toBe(existing)
})
it('lets a terminal snapshot replace the active estimate', () => {
const terminal = {
response_time_ms: 5200,
response_time_updated_at: null,
}
expect(mergeUsageRecordResponseTiming({
response_time_ms: 5500,
response_time_updated_at: '2026-07-17T12:00:06Z',
}, terminal, { preferNext: true })).toBe(terminal)
})
})
describe('mergeUsageRecordErrorMessage', () => {
const cyberMessage = 'This content was flagged for possible cybersecurity risk. Join the Trusted Access for Cyber program: https://chatgpt.com/cyber'
it('keeps an authoritative Cyber Policy message when trace reports a generic error', () => {
expect(mergeUsageRecordErrorMessage(
cyberMessage,
'execution runtime stream ended with a terminal error',
)).toBe(cyberMessage)
})
it('keeps an existing error when trace omits its error message', () => {
expect(mergeUsageRecordErrorMessage(cyberMessage, undefined)).toBe(cyberMessage)
expect(mergeUsageRecordErrorMessage(cyberMessage, null)).toBe(cyberMessage)
expect(mergeUsageRecordErrorMessage(cyberMessage, ' ')).toBe(cyberMessage)
})
it('accepts a Cyber Policy message discovered by trace', () => {
expect(mergeUsageRecordErrorMessage('Request failed', cyberMessage)).toBe(cyberMessage)
})
it('updates ordinary errors when the next snapshot has a more specific message', () => {
expect(mergeUsageRecordErrorMessage('Request failed', 'rate limit exceeded'))
.toBe('rate limit exceeded')
expect(mergeUsageRecordErrorMessage(undefined, 'Request failed')).toBe('Request failed')
})
it('lets an authoritative final-candidate snapshot replace or clear Cyber', () => {
expect(mergeUsageRecordErrorMessage(
cyberMessage,
'rate limit exceeded',
{ authoritative: true },
)).toBe('rate limit exceeded')
expect(mergeUsageRecordErrorMessage(
cyberMessage,
null,
{ authoritative: true },
)).toBeUndefined()
})
})
describe('mergeUsageRecordLifecycleSnapshot', () => {
const cyberMessage = 'This content was flagged for possible cybersecurity risk. https://chatgpt.com/cyber'
it('rejects an older failed detail without changing status, code, or error', () => {
expect(mergeUsageRecordLifecycleSnapshot({
status: 'completed',
status_code: 200,
error_message: undefined,
updated_at: '2026-07-17T00:00:02Z',
}, {
status: 'failed',
statusCode: 400,
errorMessage: cyberMessage,
updatedAt: '2026-07-17T00:00:01Z',
})).toEqual({
status: 'completed',
status_code: 200,
error_message: undefined,
updated_at: '2026-07-17T00:00:02Z',
accepted: false,
})
})
it('accepts a newer completed detail and clears an earlier Cyber failure', () => {
expect(mergeUsageRecordLifecycleSnapshot({
status: 'failed',
status_code: 400,
error_message: cyberMessage,
updated_at: '2026-07-17T00:00:01Z',
}, {
status: 'completed',
statusCode: 200,
errorMessage: null,
updatedAt: '2026-07-17T00:00:02Z',
})).toEqual({
status: 'completed',
status_code: 200,
error_message: undefined,
updated_at: '2026-07-17T00:00:02Z',
accepted: true,
})
})
})
@@ -8,8 +8,8 @@ import {
} from '../service-tier'
describe('service tier facts', () => {
it('keeps requested, actual and billing tiers independent', () => {
const facts = resolveServiceTierFacts({
it('uses the final provider request tier for display and billing', () => {
const source = {
service_tier: 'priority',
actual_service_tier: 'default',
settlement: {
@@ -21,17 +21,28 @@ describe('service tier facts', () => {
},
},
},
})
}
const facts = resolveServiceTierFacts(source)
expect(facts).toEqual({ requested: 'priority', actual: 'default', billing: 'standard' })
expect(facts).toEqual({ requested: 'priority' })
expect(hasServiceTierFact(facts)).toBe(true)
})
it('does not infer billing from requested or actual tiers', () => {
expect(resolveServiceTierFacts({
service_tier: 'priority',
it('does not infer a tier from the provider response or settlement snapshot', () => {
const source = {
actual_service_tier: 'flex',
})).toEqual({ requested: 'priority', actual: 'flex', billing: null })
settlement: {
settlement_snapshot: {
pricing_snapshot: {
billing_processing_tier: 'priority',
},
},
},
}
const facts = resolveServiceTierFacts(source)
expect(facts).toEqual({ requested: null })
expect(hasServiceTierFact(facts)).toBe(false)
})
it('normalizes only non-empty string facts', () => {
@@ -0,0 +1,70 @@
const CYBER_POLICY_TEXT_MARKERS = [
'possible cybersecurity risk',
'trusted access for cyber',
'chatgpt.com/cyber',
]
const CYBER_POLICY_CLASSIFIER_FIELDS = [
'code',
'type',
'category',
'reason',
] as const
const CYBER_ERROR_OBJECT_FIELDS = [
'error',
'errors',
'message',
'error_message',
'detail',
'body',
'response_body',
'upstream_error',
'failure_summary',
] as const
function normalizeCyberClassifier(value: string): string {
return value.trim().toLowerCase().replace(/[\s-]+/g, '_')
}
function isCyberPolicyClassifier(value: unknown): boolean {
if (typeof value !== 'string') return false
const normalized = normalizeCyberClassifier(value)
if (normalized === 'cyber' || normalized === 'cyber_policy') return true
// Providers have used nearby classifier spellings while keeping the same
// structured error contract. Keep this deliberately narrower than a generic
// substring check so ordinary cybersecurity content is not badged.
return /^(?:cyber|cybersecurity)_(?:policy|safety|risk)(?:_(?:violation|error|refusal|blocked))?$/.test(normalized)
}
function isCyberPolicyText(value: string): boolean {
const normalized = value.trim().toLowerCase()
if (!normalized) return false
return CYBER_POLICY_TEXT_MARKERS.some(marker => normalized.includes(marker))
|| /["'](?:code|type|category|reason)["']\s*:\s*["'](?:cyber|cyber[-_ ]policy|cyber[-_ ]safety|cybersecurity[-_ ](?:policy|risk))["']/i.test(normalized)
}
function detectCyberPolicyError(value: unknown, seen: WeakSet<object>): boolean {
if (typeof value === 'string') return isCyberPolicyText(value)
if (value === null || typeof value !== 'object') return false
if (seen.has(value)) return false
seen.add(value)
if (Array.isArray(value)) return value.some(item => detectCyberPolicyError(item, seen))
const record = value as Record<string, unknown>
if (CYBER_POLICY_CLASSIFIER_FIELDS.some(field => isCyberPolicyClassifier(record[field]))) {
return true
}
return CYBER_ERROR_OBJECT_FIELDS.some(field => detectCyberPolicyError(record[field], seen))
}
/**
* Detects the provider's Cyber Policy refusal without treating generic HTTP 400,
* invalid_request, or ordinary uses of the word "cyber" as policy failures.
*/
export function isCyberPolicyError(value: unknown): boolean {
return detectCyberPolicyError(value, new WeakSet<object>())
}
+157 -1
View File
@@ -1,10 +1,63 @@
import type { UsageRecord } from '../types'
import type { RequestStatus, UsageRecord } from '../types'
import { isCyberPolicyError } from './cyberError'
export type UsageRecordStreamResolution = Pick<
UsageRecord,
'id' | 'is_stream' | 'upstream_is_stream' | 'client_requested_stream' | 'client_is_stream'
>
export type UsageRecordResponseTiming = Pick<
UsageRecord,
'response_time_ms' | 'response_time_updated_at'
>
function finiteNonNegativeDurationMs(value: number | null | undefined): number | null {
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null
}
export function parseUsageTimestampMs(value: string | null | undefined): number | null {
if (!value) return null
const normalized = /(?:Z|[+-]\d{2}:\d{2})$/i.test(value) ? value : `${value}Z`
const timestampMs = new Date(normalized).getTime()
return Number.isFinite(timestampMs) ? timestampMs : null
}
/**
* Merge a live response duration together with the timestamp that anchors it.
*
* These fields form one clock snapshot: active elapsed time is projected as
* `response_time_ms + (now - response_time_updated_at)`. Merging the larger
* duration with a newer timestamp can therefore manufacture a shorter clock
* that never existed. Keep the pair atomic and, while active, retain whichever
* snapshot projects the larger elapsed value.
*/
export function mergeUsageRecordResponseTiming(
existing: UsageRecordResponseTiming,
next: UsageRecordResponseTiming,
options: { preferNext?: boolean } = {},
): UsageRecordResponseTiming {
const existingDurationMs = finiteNonNegativeDurationMs(existing.response_time_ms)
const nextDurationMs = finiteNonNegativeDurationMs(next.response_time_ms)
if (nextDurationMs == null) return existing
if (options.preferNext || existingDurationMs == null) return next
const existingUpdatedAtMs = parseUsageTimestampMs(existing.response_time_updated_at)
const nextUpdatedAtMs = parseUsageTimestampMs(next.response_time_updated_at)
if (existingUpdatedAtMs != null && nextUpdatedAtMs != null) {
const existingStartedAtMs = existingUpdatedAtMs - existingDurationMs
const nextStartedAtMs = nextUpdatedAtMs - nextDurationMs
return nextStartedAtMs <= existingStartedAtMs ? next : existing
}
// An anchored snapshot is safer than an unanchored duration for a live clock.
if (existingUpdatedAtMs != null) return existing
if (nextUpdatedAtMs != null) return next
return nextDurationMs >= existingDurationMs ? next : existing
}
export function mergeUsageRecordFirstByteTimeMs(
existingValue: number | null | undefined,
nextValue: number | null | undefined
@@ -29,6 +82,109 @@ export function mergeUsageRecordFirstByteTimeMs(
return existingValue == null ? existingValue : undefined
}
export function mergeUsageRecordErrorMessage(
existingValue: string | null | undefined,
nextValue: string | null | undefined,
options: { authoritative?: boolean } = {},
): string | undefined {
const existing = typeof existingValue === 'string' && existingValue.trim()
? existingValue
: undefined
const next = typeof nextValue === 'string' && nextValue.trim()
? nextValue
: undefined
// Complete list/active snapshots describe the current final candidate. They
// must be able to replace *and clear* an error left by an earlier candidate.
if (options.authoritative) return next
if (!next) return existing
// Detail/trace snapshots may carry a generic runtime message. Do not let that
// downgrade a provider Cyber Policy refusal already resolved by the usage list.
if (isCyberPolicyError(existing) && !isCyberPolicyError(next)) return existing
return next
}
export type UsageRecordLifecycleSnapshot = Pick<
UsageRecord,
'status' | 'status_code' | 'error_message' | 'updated_at'
>
export type UsageRecordLifecycleUpdate = {
status?: RequestStatus
statusCode?: number | null
errorMessage?: string | null
updatedAt?: string | null
}
/**
* Merge the sparse lifecycle state emitted by the detail drawer.
*
* Status, status code and error belong to one snapshot. If a detail response
* is older (or its status would regress), none of those fields may leak into
* the newer row. A completed/cancelled or explicitly newer terminal snapshot
* is authoritative for errors; a same-snapshot generic failure still keeps a
* provider Cyber refusal already known by the list.
*/
export function mergeUsageRecordLifecycleSnapshot(
existing: UsageRecordLifecycleSnapshot,
update: UsageRecordLifecycleUpdate,
): UsageRecordLifecycleSnapshot & { accepted: boolean } {
const statusPriority: Record<RequestStatus, number> = {
pending: 0,
streaming: 1,
completed: 2,
failed: 2,
cancelled: 2,
}
const existingUpdatedAtMs = parseUsageTimestampMs(existing.updated_at)
const nextUpdatedAtMs = parseUsageTimestampMs(update.updatedAt)
const nextSnapshotIsOlder = existingUpdatedAtMs != null &&
nextUpdatedAtMs != null &&
nextUpdatedAtMs < existingUpdatedAtMs
const currentRank = existing.status ? statusPriority[existing.status] : -1
const nextRank = update.status ? statusPriority[update.status] : -1
const statusAccepted = update.status != null &&
!nextSnapshotIsOlder &&
nextRank >= currentRank
const accepted = !nextSnapshotIsOlder && (update.status == null || statusAccepted)
if (!accepted) {
return { ...existing, accepted: false }
}
const terminalSnapshotIsStrictlyNewer = statusAccepted &&
(update.status === 'completed' || update.status === 'failed' || update.status === 'cancelled') &&
existingUpdatedAtMs != null &&
nextUpdatedAtMs != null &&
nextUpdatedAtMs > existingUpdatedAtMs
const errorIsAuthoritative = statusAccepted && (
update.status === 'completed' ||
update.status === 'cancelled' ||
terminalSnapshotIsStrictlyNewer
)
const hasStatusCode = Object.prototype.hasOwnProperty.call(update, 'statusCode')
const hasErrorMessage = Object.prototype.hasOwnProperty.call(update, 'errorMessage')
return {
status: statusAccepted ? update.status : existing.status,
status_code: hasStatusCode ? (update.statusCode ?? undefined) : existing.status_code,
error_message: hasErrorMessage
? mergeUsageRecordErrorMessage(
existing.error_message,
update.errorMessage,
{ authoritative: errorIsAuthoritative },
)
: existing.error_message,
updated_at: typeof update.updatedAt === 'string'
? update.updatedAt
: existing.updated_at,
accepted: true,
}
}
export function syncUsageRecordStreamResolution(
records: UsageRecord[],
resolved: UsageRecordStreamResolution
@@ -1,30 +1,26 @@
export interface ServiceTierFacts {
requested: string | null
actual: string | null
billing: string | null
}
export interface ServiceTierFactSource {
service_tier?: unknown
actual_service_tier?: unknown
settlement?: unknown
}
export function resolveServiceTierFacts(
source: ServiceTierFactSource | null | undefined,
): ServiceTierFacts {
const settlement = asRecord(source?.settlement)
const settlementSnapshot = asRecord(settlement?.settlement_snapshot)
const pricingSnapshot = asRecord(settlementSnapshot?.pricing_snapshot)
// The processing tier is an input-side fact: it must come from the final
// request body sent to the provider. Response-advertised tiers and old
// settlement snapshots can describe a different/legacy value, so they are
// deliberately not consulted here. The billing display uses this same
// authoritative request tier.
return {
requested: normalizeServiceTierFact(source?.service_tier),
actual: normalizeServiceTierFact(source?.actual_service_tier),
billing: normalizeServiceTierFact(pricingSnapshot?.billing_processing_tier),
}
}
export function hasServiceTierFact(facts: ServiceTierFacts): boolean {
return facts.requested !== null || facts.actual !== null || facts.billing !== null
return facts.requested !== null
}
export function normalizeServiceTierFact(value: unknown): string | null {
@@ -47,9 +43,3 @@ export function formatServiceTierFact(value: unknown): string | null {
? 'Fast'
: normalized
}
function asRecord(value: unknown): Record<string, unknown> | null {
return value !== null && typeof value === 'object' && !Array.isArray(value)
? value as Record<string, unknown>
: null
}