mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-12 22:20:19 +08:00
chore: update gateway pressure observability
This commit is contained in:
elky
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env node
|
||||
const path = require('node:path')
|
||||
const { spawnSync } = require('node:child_process')
|
||||
|
||||
const reportPath = process.argv[2] || '/tmp/aether_gateway_pressure_s1_1k.json'
|
||||
const checker = path.join(__dirname, 'check_gateway_stage_report.js')
|
||||
const result = spawnSync(process.execPath, [checker, '--stage', 'S1', reportPath], {
|
||||
stdio: 'inherit',
|
||||
})
|
||||
|
||||
process.exit(result.status ?? 1)
|
||||
+452
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env node
|
||||
const http = require('node:http')
|
||||
const https = require('node:https')
|
||||
|
||||
const DEFAULT_STAGE = 'S1'
|
||||
const DEFAULT_GATEWAY_BASE_URL = 'http://127.0.0.1:8084'
|
||||
const DEFAULT_MOCK_UPSTREAM_METRICS_URL = 'http://127.0.0.1:18181/metrics'
|
||||
|
||||
const REQUIRED_M4_METRICS = [
|
||||
'gateway_process_open_fds',
|
||||
'gateway_process_fd_limit',
|
||||
'gateway_process_fd_usage_basis_points',
|
||||
'gateway_process_threads',
|
||||
'gateway_process_socket_fds',
|
||||
'gateway_process_tcp_established_connections',
|
||||
'gateway_host_tcp_established_connections',
|
||||
'gateway_network_observability_available',
|
||||
'gateway_network_received_bytes_total',
|
||||
'gateway_background_tasks_active',
|
||||
'gateway_background_tasks_unexpected_exits_total',
|
||||
'gateway_tokio_runtime_observability_available',
|
||||
'gateway_tokio_runtime_workers',
|
||||
'gateway_allocator_observability_available',
|
||||
'postgres_observability_available',
|
||||
'postgres_wal_observability_available',
|
||||
'postgres_checkpoint_observability_available',
|
||||
'postgres_statement_observability_available',
|
||||
'redis_runtime_enabled',
|
||||
'redis_runtime_lane_command_latency_ms_max',
|
||||
'usage_runtime_queue_worker_read_batches_total',
|
||||
'usage_runtime_queue_worker_acked_entries_total',
|
||||
'usage_counter_outbox_flush_batches_total',
|
||||
'usage_counter_outbox_cleanup_rows_total',
|
||||
'request_candidate_queue_flush_batches_total',
|
||||
'request_candidate_queue_flush_sql_ops_total',
|
||||
]
|
||||
|
||||
function parseArgs(argv) {
|
||||
const gatewayBaseUrl = process.env.GATEWAY_BASE_URL || DEFAULT_GATEWAY_BASE_URL
|
||||
const options = {
|
||||
stage: process.env.PRESSURE_STAGE || DEFAULT_STAGE,
|
||||
gatewayBaseUrl,
|
||||
healthUrl: `${gatewayBaseUrl.replace(/\/$/, '')}/_gateway/health`,
|
||||
metricsUrl: process.env.METRICS_URL || `${gatewayBaseUrl.replace(/\/$/, '')}/_gateway/metrics`,
|
||||
targetUrl: process.env.TARGET_URL || `${gatewayBaseUrl.replace(/\/$/, '')}/v1/chat/completions`,
|
||||
mockUpstreamMetricsUrl:
|
||||
process.env.PRESSURE_MOCK_UPSTREAM_METRICS_URL || DEFAULT_MOCK_UPSTREAM_METRICS_URL,
|
||||
timeoutMs: numberEnv('PRESSURE_PREFLIGHT_TIMEOUT_MS') ?? 5000,
|
||||
requireAuth: boolEnv('PRESSURE_REQUIRE_AUTH', true),
|
||||
requireM4Metrics: boolEnv('PRESSURE_REQUIRE_M4_METRICS', true),
|
||||
requireMockUpstream: boolEnv('PRESSURE_REQUIRE_MOCK_UPSTREAM', true),
|
||||
apiKeyFile:
|
||||
process.env.AETHER_API_KEY_FILE ||
|
||||
process.env.API_KEY_FILE ||
|
||||
process.env.PRESSURE_API_KEY_FILE ||
|
||||
'',
|
||||
apiKeyListFile:
|
||||
process.env.AETHER_API_KEY_LIST_FILE ||
|
||||
process.env.API_KEY_LIST_FILE ||
|
||||
process.env.PRESSURE_API_KEY_LIST_FILE ||
|
||||
'',
|
||||
}
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index]
|
||||
switch (arg) {
|
||||
case '--stage':
|
||||
options.stage = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--gateway-base-url':
|
||||
options.gatewayBaseUrl = requireValue(argv, ++index, arg)
|
||||
options.healthUrl = `${options.gatewayBaseUrl.replace(/\/$/, '')}/_gateway/health`
|
||||
if (!process.env.METRICS_URL) {
|
||||
options.metricsUrl = `${options.gatewayBaseUrl.replace(/\/$/, '')}/_gateway/metrics`
|
||||
}
|
||||
if (!process.env.TARGET_URL) {
|
||||
options.targetUrl = `${options.gatewayBaseUrl.replace(/\/$/, '')}/v1/chat/completions`
|
||||
}
|
||||
break
|
||||
case '--health-url':
|
||||
options.healthUrl = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--metrics-url':
|
||||
options.metricsUrl = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--target-url':
|
||||
options.targetUrl = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--mock-upstream-metrics-url':
|
||||
options.mockUpstreamMetricsUrl = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--timeout-ms':
|
||||
options.timeoutMs = parsePositiveInteger(requireValue(argv, ++index, arg), arg)
|
||||
break
|
||||
case '--require-auth':
|
||||
options.requireAuth = true
|
||||
break
|
||||
case '--skip-auth':
|
||||
options.requireAuth = false
|
||||
break
|
||||
case '--api-key-file':
|
||||
options.apiKeyFile = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--api-key-list-file':
|
||||
options.apiKeyListFile = requireValue(argv, ++index, arg)
|
||||
break
|
||||
case '--require-m4-metrics':
|
||||
options.requireM4Metrics = true
|
||||
break
|
||||
case '--skip-m4-metrics':
|
||||
options.requireM4Metrics = false
|
||||
break
|
||||
case '--require-mock-upstream':
|
||||
options.requireMockUpstream = true
|
||||
break
|
||||
case '--skip-mock-upstream':
|
||||
options.requireMockUpstream = false
|
||||
break
|
||||
case '--help':
|
||||
case '-h':
|
||||
printHelp()
|
||||
process.exit(0)
|
||||
default:
|
||||
throw new Error(`unknown option: ${arg}`)
|
||||
}
|
||||
}
|
||||
|
||||
options.stage = String(options.stage).trim().toUpperCase()
|
||||
return options
|
||||
}
|
||||
|
||||
function numberEnv(name) {
|
||||
const value = process.env[name]
|
||||
if (value == null || value === '') {
|
||||
return undefined
|
||||
}
|
||||
return parsePositiveInteger(value, name)
|
||||
}
|
||||
|
||||
function boolEnv(name, fallback) {
|
||||
const value = process.env[name]
|
||||
if (value == null || value === '') {
|
||||
return fallback
|
||||
}
|
||||
return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase())
|
||||
}
|
||||
|
||||
function requireValue(argv, index, option) {
|
||||
const value = argv[index]
|
||||
if (!value || value.startsWith('--')) {
|
||||
throw new Error(`${option} requires a value`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
const number = Number(value)
|
||||
if (!Number.isInteger(number) || number <= 0) {
|
||||
throw new Error(`${name} must be a positive integer, got ${value}`)
|
||||
}
|
||||
return number
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: tools/pressure/check_gateway_stage_preflight.js [options]
|
||||
|
||||
Checks whether a gateway is ready to run staged mock streaming pressure tests.
|
||||
The script never prints auth header or API key values.
|
||||
|
||||
Options:
|
||||
--stage S1|S2|S3|S4|S5
|
||||
--gateway-base-url URL
|
||||
--health-url URL
|
||||
--metrics-url URL
|
||||
--target-url URL
|
||||
--mock-upstream-metrics-url URL
|
||||
--timeout-ms N
|
||||
--skip-auth
|
||||
--api-key-file PATH
|
||||
--api-key-list-file PATH
|
||||
--skip-mock-upstream
|
||||
--skip-m4-metrics
|
||||
`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2))
|
||||
const ok = []
|
||||
const failures = []
|
||||
|
||||
if (options.requireAuth) {
|
||||
if (authConfigured(options)) {
|
||||
ok.push('auth configured')
|
||||
} else {
|
||||
failures.push(
|
||||
'missing auth: set AUTH_HEADER, AETHER_API_KEY, API_KEY, AETHER_API_KEY_FILE, or AETHER_API_KEY_LIST_FILE',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (!isHttpUrl(options.targetUrl)) {
|
||||
failures.push(`target URL is not http(s): ${options.targetUrl}`)
|
||||
} else {
|
||||
ok.push(`target configured: ${options.targetUrl}`)
|
||||
}
|
||||
|
||||
let metricsText = ''
|
||||
await checkHttpText('gateway health', options.healthUrl, options.timeoutMs, ok, failures, (body) => {
|
||||
const health = parseJson(body)
|
||||
if (!health) {
|
||||
failures.push('gateway health did not return JSON')
|
||||
return
|
||||
}
|
||||
if (health.status !== 'ok') {
|
||||
failures.push(`gateway health status=${health.status ?? 'missing'}, expected ok`)
|
||||
return
|
||||
}
|
||||
ok.push('gateway health status ok')
|
||||
})
|
||||
|
||||
await checkHttpText('gateway metrics', options.metricsUrl, options.timeoutMs, ok, failures, (body) => {
|
||||
metricsText = body
|
||||
const metricNames = parseMetricNames(body)
|
||||
if (metricNames.size === 0) {
|
||||
failures.push('gateway metrics response did not contain Prometheus samples')
|
||||
return
|
||||
}
|
||||
ok.push(`gateway metrics samples available (${metricNames.size} metric names)`)
|
||||
|
||||
if (options.requireM4Metrics) {
|
||||
const missing = REQUIRED_M4_METRICS.filter((name) => !metricNames.has(name))
|
||||
if (missing.length > 0) {
|
||||
failures.push(`gateway metrics missing M4 required metrics: ${missing.join(', ')}`)
|
||||
} else {
|
||||
ok.push('gateway M4 metrics present')
|
||||
}
|
||||
|
||||
const tokioAvailable = metricMax(body, 'gateway_tokio_runtime_observability_available')
|
||||
if (tokioAvailable !== null && tokioAvailable !== 1) {
|
||||
failures.push(`gateway_tokio_runtime_observability_available=${tokioAvailable}, expected 1`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (options.requireMockUpstream) {
|
||||
await checkHttpText(
|
||||
'mock upstream metrics',
|
||||
options.mockUpstreamMetricsUrl,
|
||||
options.timeoutMs,
|
||||
ok,
|
||||
failures,
|
||||
(body) => {
|
||||
if (!body.trim()) {
|
||||
failures.push('mock upstream metrics response was empty')
|
||||
return
|
||||
}
|
||||
ok.push('mock upstream metrics available')
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
console.log(`gateway staged pressure preflight: ${options.stage}`)
|
||||
ok.forEach((line) => console.log(`OK ${line}`))
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error('FAIL preflight checks failed:')
|
||||
failures.forEach((line) => console.error(`FAIL ${line}`))
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (metricsText) {
|
||||
const dbPoolMax = metricMax(metricsText, 'database_pool_max_connections')
|
||||
const upstreamPermits = metricMax(metricsText, 'concurrency_available_permits', {
|
||||
gate: 'gateway_upstream_execution',
|
||||
})
|
||||
if (dbPoolMax !== null) {
|
||||
console.log(`OK database_pool_max_connections=${dbPoolMax}`)
|
||||
}
|
||||
if (upstreamPermits !== null) {
|
||||
console.log(`OK gateway_upstream_execution_available_permits=${upstreamPermits}`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('PASS gateway staged pressure preflight')
|
||||
}
|
||||
|
||||
function authConfigured(options) {
|
||||
if (['AUTH_HEADER', 'AETHER_API_KEY', 'API_KEY'].some((name) => {
|
||||
const value = process.env[name]
|
||||
return typeof value === 'string' && value.trim().length > 0
|
||||
})) {
|
||||
return true
|
||||
}
|
||||
return fileHasSecret(options.apiKeyFile) || fileHasSecret(options.apiKeyListFile)
|
||||
}
|
||||
|
||||
function fileHasSecret(path) {
|
||||
if (!path || !path.trim()) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
const fs = require('node:fs')
|
||||
return fs.readFileSync(path, 'utf8').trim().length > 0
|
||||
} catch (_error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function checkHttpText(label, url, timeoutMs, ok, failures, inspect) {
|
||||
if (!isHttpUrl(url)) {
|
||||
failures.push(`${label} URL is not http(s): ${url}`)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await requestText(url, timeoutMs)
|
||||
if (response.statusCode < 200 || response.statusCode >= 300) {
|
||||
failures.push(`${label} returned HTTP ${response.statusCode}`)
|
||||
return
|
||||
}
|
||||
inspect(response.body)
|
||||
} catch (error) {
|
||||
failures.push(`${label} unreachable at ${url}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
function isHttpUrl(value) {
|
||||
try {
|
||||
const url = new URL(value)
|
||||
return url.protocol === 'http:' || url.protocol === 'https:'
|
||||
} catch (_error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function requestText(urlString, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(urlString)
|
||||
const client = url.protocol === 'https:' ? https : http
|
||||
const request = client.get(
|
||||
url,
|
||||
{
|
||||
headers: {
|
||||
accept: 'text/plain, application/json;q=0.9, */*;q=0.1',
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
response.setEncoding('utf8')
|
||||
let body = ''
|
||||
response.on('data', (chunk) => {
|
||||
body += chunk
|
||||
})
|
||||
response.on('end', () => {
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
body,
|
||||
})
|
||||
})
|
||||
},
|
||||
)
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`timed out after ${timeoutMs}ms`))
|
||||
})
|
||||
request.on('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
function parseJson(value) {
|
||||
try {
|
||||
return JSON.parse(value)
|
||||
} catch (_error) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function parseMetricNames(text) {
|
||||
const names = new Set()
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line || line.startsWith('#')) {
|
||||
continue
|
||||
}
|
||||
const match = line.match(/^([a-zA-Z_:][a-zA-Z0-9_:.-]*)(?:\{|[\s])/)
|
||||
if (!match) {
|
||||
continue
|
||||
}
|
||||
addMetricName(names, match[1])
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
function addMetricName(names, name) {
|
||||
names.add(name)
|
||||
if (name.startsWith('aether-gateway_')) {
|
||||
names.add(name.slice('aether-gateway_'.length))
|
||||
}
|
||||
}
|
||||
|
||||
function metricMax(text, metricName, labels = {}) {
|
||||
let max = null
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line || line.startsWith('#')) {
|
||||
continue
|
||||
}
|
||||
const parsed = parseMetricSample(line)
|
||||
if (!parsed) {
|
||||
continue
|
||||
}
|
||||
if (parsed.name !== metricName && parsed.name !== `aether-gateway_${metricName}`) {
|
||||
continue
|
||||
}
|
||||
if (!labelsMatch(parsed.labels, labels)) {
|
||||
continue
|
||||
}
|
||||
max = max === null ? parsed.value : Math.max(max, parsed.value)
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
function parseMetricSample(line) {
|
||||
const match = line.match(/^([a-zA-Z_:][a-zA-Z0-9_:.-]*)(\{[^}]*\})?\s+(-?(?:\d+\.?\d*|\d*\.\d+)(?:[eE][+-]?\d+)?)\s*$/)
|
||||
if (!match) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
name: match[1],
|
||||
labels: parseLabels(match[2]),
|
||||
value: Number(match[3]),
|
||||
}
|
||||
}
|
||||
|
||||
function parseLabels(labelText) {
|
||||
if (!labelText) {
|
||||
return {}
|
||||
}
|
||||
const labels = {}
|
||||
const inner = labelText.slice(1, -1)
|
||||
const pattern = /([a-zA-Z_][a-zA-Z0-9_]*)="((?:\\.|[^"\\])*)"/g
|
||||
let match
|
||||
while ((match = pattern.exec(inner)) !== null) {
|
||||
labels[match[1]] = match[2].replace(/\\"/g, '"').replace(/\\\\/g, '\\')
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
function labelsMatch(actual, expected) {
|
||||
return Object.entries(expected).every(([name, value]) => actual[name] === value)
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`FAIL ${error.message}`)
|
||||
process.exit(1)
|
||||
})
|
||||
Executable
+1252
File diff suppressed because it is too large
Load Diff
+197
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env node
|
||||
const assert = require('node:assert/strict')
|
||||
const fs = require('node:fs')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const test = require('node:test')
|
||||
const { spawnSync } = require('node:child_process')
|
||||
|
||||
const checker = path.join(__dirname, 'check_gateway_stage_report.js')
|
||||
|
||||
test('realistic-stream report passes full-chain latency and throughput checks', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 1000,
|
||||
concurrency: 1000,
|
||||
throughputRps: 180,
|
||||
headersP95Ms: 120,
|
||||
firstBodyP95Ms: 350,
|
||||
p95Ms: 6000,
|
||||
p99Ms: 9000,
|
||||
})
|
||||
const result = runChecker('--stage', 'realistic-stream', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 0, result.stderr)
|
||||
assert.match(result.stdout, /REALISTIC_STREAM PASS/)
|
||||
assert.match(result.stdout, /throughput_rps=180/)
|
||||
})
|
||||
|
||||
test('tps report passes completed request throughput checks', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 750,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
const result = runChecker('--stage', 'TPS', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 0, result.stderr)
|
||||
assert.match(result.stdout, /TPS PASS/)
|
||||
})
|
||||
|
||||
test('tps report fails when throughput is below the acceptance threshold', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 499,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
const result = runChecker('--stage', 'tps', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(result.stderr, /TPS FAIL: load\.throughput_rps=499, expected >= 500/)
|
||||
})
|
||||
|
||||
test('tps report fails when settle drain does not complete', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 750,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
report.settle_drain_completed = false
|
||||
const result = runChecker('--stage', 'tps', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(result.stderr, /TPS FAIL: settle_drain_completed=false after 5000ms/)
|
||||
})
|
||||
|
||||
test('tps report fails when lifecycle enqueue drops deferred events', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 750,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
report.metrics.usage_runtime_max_lifecycle_enqueue_deferred_dropped_total = 1
|
||||
const result = runChecker('--stage', 'tps', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(
|
||||
result.stderr,
|
||||
/TPS FAIL: usage_runtime_max_lifecycle_enqueue_deferred_dropped_total=1/,
|
||||
)
|
||||
})
|
||||
|
||||
test('tps report ignores shared redis error replies when gateway lane errors are clean', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 750,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
report.metrics.redis_runtime_total_error_replies_delta = 12
|
||||
report.metrics.redis_runtime_lane_command_errors_total_delta = 0
|
||||
report.metrics.redis_runtime_lane_command_timeouts_total_delta = 0
|
||||
const result = runChecker('--stage', 'tps', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 0, result.stderr)
|
||||
})
|
||||
|
||||
test('tps report falls back to redis error replies when lane metrics are absent', () => {
|
||||
const report = reportFor({
|
||||
totalRequests: 20000,
|
||||
concurrency: 500,
|
||||
throughputRps: 750,
|
||||
headersP95Ms: 90,
|
||||
firstBodyP95Ms: 180,
|
||||
p95Ms: 900,
|
||||
p99Ms: 1600,
|
||||
})
|
||||
report.metrics.redis_runtime_total_error_replies_delta = 1
|
||||
delete report.metrics.redis_runtime_lane_command_errors_total_delta
|
||||
delete report.metrics.redis_runtime_lane_command_timeouts_total_delta
|
||||
const result = runChecker('--stage', 'tps', writeReport(report))
|
||||
|
||||
assert.equal(result.status, 1)
|
||||
assert.match(result.stderr, /TPS FAIL: redis_runtime_total_error_replies_delta=1/)
|
||||
})
|
||||
|
||||
function runChecker(...args) {
|
||||
return spawnSync(process.execPath, [checker, ...args], {
|
||||
cwd: path.resolve(__dirname, '../..'),
|
||||
encoding: 'utf8',
|
||||
})
|
||||
}
|
||||
|
||||
function writeReport(report) {
|
||||
const file = path.join(
|
||||
fs.mkdtempSync(path.join(os.tmpdir(), 'aether-stage-report-')),
|
||||
'report.json',
|
||||
)
|
||||
fs.writeFileSync(file, `${JSON.stringify(report)}\n`)
|
||||
return file
|
||||
}
|
||||
|
||||
function reportFor({
|
||||
totalRequests,
|
||||
concurrency,
|
||||
throughputRps,
|
||||
headersP95Ms,
|
||||
firstBodyP95Ms,
|
||||
p95Ms,
|
||||
p99Ms,
|
||||
}) {
|
||||
return {
|
||||
settle_after_ms: 5000,
|
||||
settle_drain_completed: true,
|
||||
settle_drain_elapsed_ms: 250,
|
||||
load: {
|
||||
response_mode: 'FullBody',
|
||||
total_requests: totalRequests,
|
||||
completed_requests: totalRequests,
|
||||
failed_requests: 0,
|
||||
concurrency,
|
||||
first_body_hold_ms: 0,
|
||||
throughput_rps: throughputRps,
|
||||
headers_p95_ms: headersP95Ms,
|
||||
first_body_p95_ms: firstBodyP95Ms,
|
||||
p95_ms: p95Ms,
|
||||
p99_ms: p99Ms,
|
||||
status_counts: { 200: totalRequests },
|
||||
error_counts: {},
|
||||
},
|
||||
metrics: {
|
||||
samples: 10,
|
||||
db_pool_max_usage_basis_points: 2500,
|
||||
db_pool_pressure_samples: 0,
|
||||
gateway_requests_max_rejected_total: 0,
|
||||
gateway_requests_distributed_max_rejected_total: 0,
|
||||
request_candidate_queue_final_depth: 0,
|
||||
request_candidate_queue_final_pending_depth: 0,
|
||||
request_candidate_queue_max_flush_failed_total: 0,
|
||||
request_candidate_queue_max_dropped_total: 0,
|
||||
request_candidate_queue_max_sync_fallback_total: 0,
|
||||
usage_runtime_max_terminal_enqueue_failed_total: 0,
|
||||
usage_runtime_max_lifecycle_enqueue_failed_total: 0,
|
||||
usage_runtime_max_lifecycle_enqueue_deferred_dropped_total: 0,
|
||||
redis_runtime_lane_command_errors_total_delta: 0,
|
||||
redis_runtime_lane_command_timeouts_total_delta: 0,
|
||||
upstream_target_max_rejected_total: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PRESSURE_STAGE="${PRESSURE_STAGE:-S1}"
|
||||
exec "$(dirname "$0")/run_gateway_mock_streaming_stage.sh"
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Gateway staged mock streaming pressure probe.
|
||||
#
|
||||
# Required auth:
|
||||
# AETHER_API_KEY_FILE=/path/to/api-key
|
||||
# or:
|
||||
# AUTH_HEADER='Authorization: Bearer <aether-api-key>'
|
||||
# or:
|
||||
# AETHER_API_KEY='<aether-api-key>'
|
||||
#
|
||||
# Common settings:
|
||||
# PRESSURE_STAGE=S1|S2|S3|S4|S5
|
||||
# GATEWAY_BASE_URL=http://127.0.0.1:8084
|
||||
# TARGET_URL=http://127.0.0.1:8084/v1/chat/completions
|
||||
# METRICS_URL=http://127.0.0.1:8084/_gateway/metrics
|
||||
# PRESSURE_MODEL=gpt-5-mini
|
||||
# PRESSURE_RESPONSE_MODE=first-body-byte
|
||||
# PRESSURE_CARGO_PROFILE=release
|
||||
|
||||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(cd -- "$script_dir/../.." && pwd)"
|
||||
|
||||
PRESSURE_STAGE="${PRESSURE_STAGE:-S1}"
|
||||
PRESSURE_STAGE="$(printf '%s' "$PRESSURE_STAGE" | tr '[:lower:]' '[:upper:]')"
|
||||
|
||||
case "$PRESSURE_STAGE" in
|
||||
S1)
|
||||
default_requests=1000
|
||||
default_concurrency=1000
|
||||
default_hold_ms=600000
|
||||
default_timeout_ms=720000
|
||||
default_start_ramp_ms=10000
|
||||
default_output=/tmp/aether_gateway_pressure_s1_1k.json
|
||||
;;
|
||||
S2)
|
||||
default_requests=3000
|
||||
default_concurrency=3000
|
||||
default_hold_ms=900000
|
||||
default_timeout_ms=1080000
|
||||
default_start_ramp_ms=30000
|
||||
default_output=/tmp/aether_gateway_pressure_s2_3k.json
|
||||
;;
|
||||
S3)
|
||||
default_requests=6000
|
||||
default_concurrency=6000
|
||||
default_hold_ms=1800000
|
||||
default_timeout_ms=1980000
|
||||
default_start_ramp_ms=60000
|
||||
default_output=/tmp/aether_gateway_pressure_s3_6k.json
|
||||
;;
|
||||
S4)
|
||||
default_requests=10000
|
||||
default_concurrency=10000
|
||||
default_hold_ms=1800000
|
||||
default_timeout_ms=2100000
|
||||
default_start_ramp_ms=90000
|
||||
default_output=/tmp/aether_gateway_pressure_s4_10k.json
|
||||
;;
|
||||
S5)
|
||||
default_requests=10000
|
||||
default_concurrency=10000
|
||||
default_hold_ms=7200000
|
||||
default_timeout_ms=7500000
|
||||
default_start_ramp_ms=120000
|
||||
default_output=/tmp/aether_gateway_pressure_s5_10k_soak.json
|
||||
;;
|
||||
*)
|
||||
echo "unsupported PRESSURE_STAGE=$PRESSURE_STAGE; expected S1, S2, S3, S4, or S5" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
GATEWAY_BASE_URL="${GATEWAY_BASE_URL:-http://127.0.0.1:8084}"
|
||||
TARGET_URL="${TARGET_URL:-${GATEWAY_BASE_URL%/}/v1/chat/completions}"
|
||||
METRICS_URL="${METRICS_URL:-${GATEWAY_BASE_URL%/}/_gateway/metrics}"
|
||||
PRESSURE_REQUESTS="${PRESSURE_REQUESTS:-$default_requests}"
|
||||
PRESSURE_CONCURRENCY="${PRESSURE_CONCURRENCY:-$default_concurrency}"
|
||||
PRESSURE_TIMEOUT_MS="${PRESSURE_TIMEOUT_MS:-$default_timeout_ms}"
|
||||
PRESSURE_CONNECT_TIMEOUT_MS="${PRESSURE_CONNECT_TIMEOUT_MS:-30000}"
|
||||
PRESSURE_SAMPLE_INTERVAL_MS="${PRESSURE_SAMPLE_INTERVAL_MS:-500}"
|
||||
PRESSURE_SETTLE_AFTER_MS="${PRESSURE_SETTLE_AFTER_MS:-2000}"
|
||||
PRESSURE_START_RAMP_MS="${PRESSURE_START_RAMP_MS:-$default_start_ramp_ms}"
|
||||
PRESSURE_FIRST_BODY_HOLD_MS="${PRESSURE_FIRST_BODY_HOLD_MS:-$default_hold_ms}"
|
||||
PRESSURE_METHOD="${PRESSURE_METHOD:-POST}"
|
||||
PRESSURE_RESPONSE_MODE="${PRESSURE_RESPONSE_MODE:-first-body-byte}"
|
||||
PRESSURE_CARGO_PROFILE="${PRESSURE_CARGO_PROFILE:-release}"
|
||||
PRESSURE_MODEL="${PRESSURE_MODEL:-gpt-5-mini}"
|
||||
OUTPUT="${OUTPUT:-$default_output}"
|
||||
api_key_file="${AETHER_API_KEY_FILE:-${API_KEY_FILE:-${PRESSURE_API_KEY_FILE:-}}}"
|
||||
stage_lower="$(printf '%s' "$PRESSURE_STAGE" | tr '[:upper:]' '[:lower:]')"
|
||||
PRESSURE_BODY_FILE="${PRESSURE_BODY_FILE:-/tmp/aether-pressure-${stage_lower}-mock-streaming-request.json}"
|
||||
|
||||
if [[ -z "${AUTH_HEADER:-}" ]]; then
|
||||
if [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
:
|
||||
elif [[ -n "${AETHER_API_KEY:-}" ]]; then
|
||||
AUTH_HEADER="Authorization: Bearer ${AETHER_API_KEY}"
|
||||
elif [[ -n "${API_KEY:-}" ]]; then
|
||||
AUTH_HEADER="Authorization: Bearer ${API_KEY}"
|
||||
else
|
||||
echo "missing auth: set AETHER_API_KEY_FILE, AUTH_HEADER, or AETHER_API_KEY before running gateway staged pressure" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${PRESSURE_BODY:-}" && ! -s "$PRESSURE_BODY_FILE" ]]; then
|
||||
cat >"$PRESSURE_BODY_FILE" <<JSON
|
||||
{"model":"${PRESSURE_MODEL}","messages":[{"role":"user","content":"ping"}],"stream":true}
|
||||
JSON
|
||||
fi
|
||||
|
||||
args=(run)
|
||||
case "$PRESSURE_CARGO_PROFILE" in
|
||||
release)
|
||||
args+=(--release)
|
||||
;;
|
||||
debug)
|
||||
;;
|
||||
*)
|
||||
echo "unsupported PRESSURE_CARGO_PROFILE=$PRESSURE_CARGO_PROFILE; expected release or debug" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
args+=(
|
||||
-p aether-testkit --bin gateway_pressure_probe --
|
||||
--url "$TARGET_URL"
|
||||
--metrics-url "$METRICS_URL"
|
||||
--requests "$PRESSURE_REQUESTS"
|
||||
--concurrency "$PRESSURE_CONCURRENCY"
|
||||
--timeout-ms "$PRESSURE_TIMEOUT_MS"
|
||||
--connect-timeout-ms "$PRESSURE_CONNECT_TIMEOUT_MS"
|
||||
--sample-interval-ms "$PRESSURE_SAMPLE_INTERVAL_MS"
|
||||
--settle-after-ms "$PRESSURE_SETTLE_AFTER_MS"
|
||||
--start-ramp-ms "$PRESSURE_START_RAMP_MS"
|
||||
--first-body-hold-ms "$PRESSURE_FIRST_BODY_HOLD_MS"
|
||||
--method "$PRESSURE_METHOD"
|
||||
--response-mode "$PRESSURE_RESPONSE_MODE"
|
||||
--output "$OUTPUT"
|
||||
)
|
||||
|
||||
if [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
args+=(--api-key-file "$api_key_file")
|
||||
else
|
||||
args+=(--header "$AUTH_HEADER")
|
||||
fi
|
||||
|
||||
if [[ -n "${EXTRA_HEADERS:-}" ]]; then
|
||||
while IFS= read -r header; do
|
||||
[[ -z "$header" ]] && continue
|
||||
args+=(--header "$header")
|
||||
done <<< "$EXTRA_HEADERS"
|
||||
else
|
||||
args+=(--header "Content-Type: application/json")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_CLIENT_SHARDS:-}" ]]; then
|
||||
args+=(--client-shards "$PRESSURE_CLIENT_SHARDS")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_POOL_MAX_IDLE_PER_HOST:-}" ]]; then
|
||||
args+=(--pool-max-idle-per-host "$PRESSURE_POOL_MAX_IDLE_PER_HOST")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_WARMUP_CONNECTIONS:-}" ]]; then
|
||||
args+=(--warmup-connections "$PRESSURE_WARMUP_CONNECTIONS")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_WARMUP_URL:-}" ]]; then
|
||||
args+=(--warmup-url "$PRESSURE_WARMUP_URL")
|
||||
fi
|
||||
|
||||
if [[ "${PRESSURE_HTTP1_ONLY:-false}" == "true" ]]; then
|
||||
args+=(--http1-only)
|
||||
fi
|
||||
|
||||
if [[ "${PRESSURE_HTTP2_PRIOR_KNOWLEDGE:-false}" == "true" ]]; then
|
||||
args+=(--http2-prior-knowledge)
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_BODY:-}" ]]; then
|
||||
args+=(--body "$PRESSURE_BODY")
|
||||
else
|
||||
args+=(--body-file "$PRESSURE_BODY_FILE")
|
||||
fi
|
||||
|
||||
metrics_before="${OUTPUT%.json}.metrics.before.prom"
|
||||
metrics_after="${OUTPUT%.json}.metrics.after.prom"
|
||||
|
||||
if [[ "${PRESSURE_PREFLIGHT:-true}" == "true" ]]; then
|
||||
preflight_args=(
|
||||
--stage "$PRESSURE_STAGE"
|
||||
--gateway-base-url "$GATEWAY_BASE_URL"
|
||||
--target-url "$TARGET_URL"
|
||||
--metrics-url "$METRICS_URL"
|
||||
)
|
||||
if [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
preflight_args+=(--api-key-file "$api_key_file")
|
||||
fi
|
||||
"$script_dir/check_gateway_stage_preflight.js" \
|
||||
"${preflight_args[@]}"
|
||||
fi
|
||||
|
||||
echo "running $PRESSURE_STAGE gateway mock streaming pressure probe"
|
||||
echo " target: $TARGET_URL"
|
||||
echo " metrics: $METRICS_URL"
|
||||
echo " requests: $PRESSURE_REQUESTS"
|
||||
echo " concurrency: $PRESSURE_CONCURRENCY"
|
||||
echo " hold ms: $PRESSURE_FIRST_BODY_HOLD_MS"
|
||||
echo " ramp ms: $PRESSURE_START_RAMP_MS"
|
||||
echo " settle ms: $PRESSURE_SETTLE_AFTER_MS"
|
||||
echo " response mode: $PRESSURE_RESPONSE_MODE"
|
||||
echo " cargo: $PRESSURE_CARGO_PROFILE"
|
||||
echo " output: $OUTPUT"
|
||||
|
||||
if [[ "${PRESSURE_CAPTURE_METRICS_SNAPSHOTS:-true}" == "true" ]]; then
|
||||
curl -fsS "$METRICS_URL" >"$metrics_before" || true
|
||||
fi
|
||||
|
||||
# Use quiet cargo output so sensitive header values are not echoed back as part
|
||||
# of Cargo's `Running ...` command line.
|
||||
(cd "$repo_root" && cargo -q "${args[@]}")
|
||||
|
||||
if [[ "${PRESSURE_CAPTURE_METRICS_SNAPSHOTS:-true}" == "true" ]]; then
|
||||
curl -fsS "$METRICS_URL" >"$metrics_after" || true
|
||||
echo "metrics snapshots written to:"
|
||||
echo " before: $metrics_before"
|
||||
echo " after: $metrics_after"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "$PRESSURE_STAGE pressure report written to $OUTPUT"
|
||||
|
||||
if [[ "${PRESSURE_CHECK_REPORT:-true}" == "true" ]]; then
|
||||
"$script_dir/check_gateway_stage_report.js" --stage "$PRESSURE_STAGE" "$OUTPUT"
|
||||
fi
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Gateway realistic profile pressure probe.
|
||||
#
|
||||
# Profiles:
|
||||
# realistic-stream: full-body streaming; use with mock upstream chunks/delay/payload set to realistic values.
|
||||
# tps: no artificial hold; measures completed request throughput through auth + DB/Redis + usage/counter paths.
|
||||
#
|
||||
# Suggested mock upstream for realistic-stream:
|
||||
# cargo run --release -p aether-testkit --bin mock_openai_upstream -- \
|
||||
# --bind 127.0.0.1:18181 --chunks 80 --first-byte-delay-ms 150 --chunk-delay-ms 50 --payload-bytes 128
|
||||
#
|
||||
# Suggested mock upstream for tps:
|
||||
# cargo run --release -p aether-testkit --bin mock_openai_upstream -- \
|
||||
# --bind 127.0.0.1:18181 --chunks 8 --first-byte-delay-ms 20 --chunk-delay-ms 5 --payload-bytes 64
|
||||
#
|
||||
# Required auth:
|
||||
# AETHER_API_KEY_FILE=/path/to/api-key
|
||||
# or:
|
||||
# AUTH_HEADER='Authorization: Bearer <aether-api-key>'
|
||||
# or:
|
||||
# AETHER_API_KEY='<aether-api-key>'
|
||||
|
||||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(cd -- "$script_dir/../.." && pwd)"
|
||||
|
||||
PROFILE="${PRESSURE_PROFILE:-${1:-realistic-stream}}"
|
||||
PROFILE="$(printf '%s' "$PROFILE" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
case "$PROFILE" in
|
||||
realistic-stream)
|
||||
default_requests=1000
|
||||
default_concurrency=1000
|
||||
default_timeout_ms=300000
|
||||
default_start_ramp_ms=10000
|
||||
default_response_mode=full
|
||||
default_settle_after_ms=5000
|
||||
default_output=/tmp/aether_gateway_realistic_stream_1k.json
|
||||
;;
|
||||
tps)
|
||||
default_requests=20000
|
||||
default_concurrency=500
|
||||
default_timeout_ms=180000
|
||||
default_start_ramp_ms=5000
|
||||
default_response_mode=full
|
||||
default_settle_after_ms=5000
|
||||
default_output=/tmp/aether_gateway_tps_20k_c500.json
|
||||
;;
|
||||
*)
|
||||
echo "unsupported PRESSURE_PROFILE=$PROFILE; expected realistic-stream or tps" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
GATEWAY_BASE_URL="${GATEWAY_BASE_URL:-http://127.0.0.1:8084}"
|
||||
TARGET_URL="${TARGET_URL:-${GATEWAY_BASE_URL%/}/v1/chat/completions}"
|
||||
METRICS_URL="${METRICS_URL:-${GATEWAY_BASE_URL%/}/_gateway/metrics}"
|
||||
PRESSURE_REQUESTS="${PRESSURE_REQUESTS:-$default_requests}"
|
||||
PRESSURE_CONCURRENCY="${PRESSURE_CONCURRENCY:-$default_concurrency}"
|
||||
PRESSURE_TIMEOUT_MS="${PRESSURE_TIMEOUT_MS:-$default_timeout_ms}"
|
||||
PRESSURE_CONNECT_TIMEOUT_MS="${PRESSURE_CONNECT_TIMEOUT_MS:-30000}"
|
||||
PRESSURE_SAMPLE_INTERVAL_MS="${PRESSURE_SAMPLE_INTERVAL_MS:-500}"
|
||||
PRESSURE_SETTLE_AFTER_MS="${PRESSURE_SETTLE_AFTER_MS:-$default_settle_after_ms}"
|
||||
PRESSURE_START_RAMP_MS="${PRESSURE_START_RAMP_MS:-$default_start_ramp_ms}"
|
||||
PRESSURE_FIRST_BODY_HOLD_MS="${PRESSURE_FIRST_BODY_HOLD_MS:-0}"
|
||||
PRESSURE_METHOD="${PRESSURE_METHOD:-POST}"
|
||||
PRESSURE_RESPONSE_MODE="${PRESSURE_RESPONSE_MODE:-$default_response_mode}"
|
||||
PRESSURE_CARGO_PROFILE="${PRESSURE_CARGO_PROFILE:-release}"
|
||||
PRESSURE_MODEL="${PRESSURE_MODEL:-gpt-5-mini}"
|
||||
OUTPUT="${OUTPUT:-$default_output}"
|
||||
api_key_file="${AETHER_API_KEY_FILE:-${API_KEY_FILE:-${PRESSURE_API_KEY_FILE:-}}}"
|
||||
api_key_list_file="${AETHER_API_KEY_LIST_FILE:-${API_KEY_LIST_FILE:-${PRESSURE_API_KEY_LIST_FILE:-}}}"
|
||||
PRESSURE_BODY_FILE="${PRESSURE_BODY_FILE:-/tmp/aether-pressure-${PROFILE}-request.json}"
|
||||
|
||||
if [[ -z "${AUTH_HEADER:-}" ]]; then
|
||||
if [[ -n "$api_key_list_file" && -s "$api_key_list_file" ]]; then
|
||||
:
|
||||
elif [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
:
|
||||
elif [[ -n "${AETHER_API_KEY:-}" ]]; then
|
||||
AUTH_HEADER="Authorization: Bearer ${AETHER_API_KEY}"
|
||||
elif [[ -n "${API_KEY:-}" ]]; then
|
||||
AUTH_HEADER="Authorization: Bearer ${API_KEY}"
|
||||
else
|
||||
echo "missing auth: set AETHER_API_KEY_FILE, AUTH_HEADER, or AETHER_API_KEY before running gateway realistic pressure" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${PRESSURE_BODY:-}" && ! -s "$PRESSURE_BODY_FILE" ]]; then
|
||||
cat >"$PRESSURE_BODY_FILE" <<JSON
|
||||
{"model":"${PRESSURE_MODEL}","messages":[{"role":"system","content":"You are a concise assistant."},{"role":"user","content":"Write a practical deployment checklist for a high-concurrency API gateway. Include authentication, billing, observability, rollout, rollback, and incident handling."}],"stream":true}
|
||||
JSON
|
||||
fi
|
||||
|
||||
args=(run)
|
||||
case "$PRESSURE_CARGO_PROFILE" in
|
||||
release)
|
||||
args+=(--release)
|
||||
;;
|
||||
debug)
|
||||
;;
|
||||
*)
|
||||
echo "unsupported PRESSURE_CARGO_PROFILE=$PRESSURE_CARGO_PROFILE; expected release or debug" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
args+=(
|
||||
-p aether-testkit --bin gateway_pressure_probe --
|
||||
--url "$TARGET_URL"
|
||||
--metrics-url "$METRICS_URL"
|
||||
--requests "$PRESSURE_REQUESTS"
|
||||
--concurrency "$PRESSURE_CONCURRENCY"
|
||||
--timeout-ms "$PRESSURE_TIMEOUT_MS"
|
||||
--connect-timeout-ms "$PRESSURE_CONNECT_TIMEOUT_MS"
|
||||
--sample-interval-ms "$PRESSURE_SAMPLE_INTERVAL_MS"
|
||||
--settle-after-ms "$PRESSURE_SETTLE_AFTER_MS"
|
||||
--start-ramp-ms "$PRESSURE_START_RAMP_MS"
|
||||
--first-body-hold-ms "$PRESSURE_FIRST_BODY_HOLD_MS"
|
||||
--method "$PRESSURE_METHOD"
|
||||
--response-mode "$PRESSURE_RESPONSE_MODE"
|
||||
--output "$OUTPUT"
|
||||
)
|
||||
|
||||
if [[ -n "$api_key_list_file" && -s "$api_key_list_file" ]]; then
|
||||
args+=(--api-key-list-file "$api_key_list_file")
|
||||
elif [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
args+=(--api-key-file "$api_key_file")
|
||||
else
|
||||
args+=(--header "$AUTH_HEADER")
|
||||
fi
|
||||
|
||||
if [[ -n "${EXTRA_HEADERS:-}" ]]; then
|
||||
while IFS= read -r header; do
|
||||
[[ -z "$header" ]] && continue
|
||||
args+=(--header "$header")
|
||||
done <<< "$EXTRA_HEADERS"
|
||||
else
|
||||
args+=(--header "Content-Type: application/json")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_CLIENT_SHARDS:-}" ]]; then
|
||||
args+=(--client-shards "$PRESSURE_CLIENT_SHARDS")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_POOL_MAX_IDLE_PER_HOST:-}" ]]; then
|
||||
args+=(--pool-max-idle-per-host "$PRESSURE_POOL_MAX_IDLE_PER_HOST")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_WARMUP_CONNECTIONS:-}" ]]; then
|
||||
args+=(--warmup-connections "$PRESSURE_WARMUP_CONNECTIONS")
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_WARMUP_URL:-}" ]]; then
|
||||
args+=(--warmup-url "$PRESSURE_WARMUP_URL")
|
||||
fi
|
||||
|
||||
if [[ "${PRESSURE_HTTP1_ONLY:-false}" == "true" ]]; then
|
||||
args+=(--http1-only)
|
||||
fi
|
||||
|
||||
if [[ "${PRESSURE_HTTP2_PRIOR_KNOWLEDGE:-false}" == "true" ]]; then
|
||||
args+=(--http2-prior-knowledge)
|
||||
fi
|
||||
|
||||
if [[ -n "${PRESSURE_BODY:-}" ]]; then
|
||||
args+=(--body "$PRESSURE_BODY")
|
||||
else
|
||||
args+=(--body-file "$PRESSURE_BODY_FILE")
|
||||
fi
|
||||
|
||||
metrics_before="${OUTPUT%.json}.metrics.before.prom"
|
||||
metrics_after="${OUTPUT%.json}.metrics.after.prom"
|
||||
|
||||
if [[ "${PRESSURE_PREFLIGHT:-true}" == "true" ]]; then
|
||||
preflight_args=(
|
||||
--stage "$PROFILE"
|
||||
--gateway-base-url "$GATEWAY_BASE_URL"
|
||||
--target-url "$TARGET_URL"
|
||||
--metrics-url "$METRICS_URL"
|
||||
)
|
||||
if [[ -n "$api_key_list_file" && -s "$api_key_list_file" ]]; then
|
||||
preflight_args+=(--api-key-list-file "$api_key_list_file")
|
||||
elif [[ -n "$api_key_file" && -s "$api_key_file" ]]; then
|
||||
preflight_args+=(--api-key-file "$api_key_file")
|
||||
fi
|
||||
"$script_dir/check_gateway_stage_preflight.js" "${preflight_args[@]}"
|
||||
fi
|
||||
|
||||
echo "running $PROFILE gateway realistic pressure probe"
|
||||
echo " target: $TARGET_URL"
|
||||
echo " metrics: $METRICS_URL"
|
||||
echo " requests: $PRESSURE_REQUESTS"
|
||||
echo " concurrency: $PRESSURE_CONCURRENCY"
|
||||
echo " response mode: $PRESSURE_RESPONSE_MODE"
|
||||
echo " hold ms: $PRESSURE_FIRST_BODY_HOLD_MS"
|
||||
echo " ramp ms: $PRESSURE_START_RAMP_MS"
|
||||
echo " settle ms: $PRESSURE_SETTLE_AFTER_MS"
|
||||
echo " cargo: $PRESSURE_CARGO_PROFILE"
|
||||
echo " output: $OUTPUT"
|
||||
|
||||
if [[ "${PRESSURE_CAPTURE_METRICS_SNAPSHOTS:-true}" == "true" ]]; then
|
||||
curl -fsS "$METRICS_URL" >"$metrics_before" || true
|
||||
fi
|
||||
|
||||
(cd "$repo_root" && cargo -q "${args[@]}")
|
||||
|
||||
if [[ "${PRESSURE_CAPTURE_METRICS_SNAPSHOTS:-true}" == "true" ]]; then
|
||||
curl -fsS "$METRICS_URL" >"$metrics_after" || true
|
||||
echo "metrics snapshots written to:"
|
||||
echo " before: $metrics_before"
|
||||
echo " after: $metrics_after"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "$PROFILE pressure report written to $OUTPUT"
|
||||
|
||||
if [[ "${PRESSURE_CHECK_REPORT:-true}" == "true" ]]; then
|
||||
check_args=(--stage "$PROFILE")
|
||||
if [[ -n "${PRESSURE_MIN_THROUGHPUT_RPS:-}" ]]; then
|
||||
check_args+=(--min-throughput-rps "$PRESSURE_MIN_THROUGHPUT_RPS")
|
||||
fi
|
||||
if [[ -n "${PRESSURE_MAX_HEADERS_P95_MS:-}" ]]; then
|
||||
check_args+=(--max-headers-p95-ms "$PRESSURE_MAX_HEADERS_P95_MS")
|
||||
fi
|
||||
if [[ -n "${PRESSURE_MAX_FIRST_BODY_P95_MS:-}" ]]; then
|
||||
check_args+=(--max-first-body-p95-ms "$PRESSURE_MAX_FIRST_BODY_P95_MS")
|
||||
fi
|
||||
if [[ -n "${PRESSURE_MAX_P95_MS:-}" ]]; then
|
||||
check_args+=(--max-p95-ms "$PRESSURE_MAX_P95_MS")
|
||||
fi
|
||||
if [[ -n "${PRESSURE_MAX_P99_MS:-}" ]]; then
|
||||
check_args+=(--max-p99-ms "$PRESSURE_MAX_P99_MS")
|
||||
fi
|
||||
if [[ -n "${PRESSURE_MAX_FIRST_BODY_HOLD_MS:-}" ]]; then
|
||||
check_args+=(--max-first-body-hold-ms "$PRESSURE_MAX_FIRST_BODY_HOLD_MS")
|
||||
fi
|
||||
"$script_dir/check_gateway_stage_report.js" "${check_args[@]}" "$OUTPUT"
|
||||
fi
|
||||
Reference in New Issue
Block a user