i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-07 18:07:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(usage): bound terminal event persistence

Browse Source 
Add end-to-end terminal admission, bounded database fallback, and observable overload handling. Preserve first-byte lifecycle state across asynchronous runtime and frontend updates.
This commit is contained in:
elky
2026-07-16 13:12:30 +08:00
parent c6d373e6aa
commit 9a47267545
10 changed files with 1952 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/execution_runtime/stream/execution.rs
+9 -29
View File
@@ -5255,16 +5255,12 @@ async fn execute_stream_from_frame_stream(
telemetry.as_ref(),
&mut usage_stream_telemetry,
) {
let usage_data = state_for_report.data.as_ref().clone();
state_for_report
.usage_runtime
.record_stream_started_direct(
&usage_data,
&lifecycle_seed_for_report,
status_code,
usage_stream_telemetry.as_ref(),
)
.await;
state_for_report.usage_runtime.record_stream_started(
state_for_report.data.as_ref(),
&lifecycle_seed_for_report,
status_code,
usage_stream_telemetry.as_ref(),
);
}
let first_data_after = usage_stream_telemetry
.as_ref()
@@ -5463,25 +5459,9 @@ async fn execute_stream_from_frame_stream(
&usage_frame_telemetry,
);
if should_refresh_stream_usage {
if usage_frame_telemetry.ttfb_ms.is_some() {
let usage_data = state_for_report.data.as_ref().clone();
state_for_report
.usage_runtime
.record_stream_started_direct(
&usage_data,
&lifecycle_seed_for_report,
status_code,
Some(&usage_frame_telemetry),
)
.await;
} else {
state_for_report.usage_runtime.record_stream_started(
state_for_report.data.as_ref(),
&lifecycle_seed_for_report,
status_code,
Some(&usage_frame_telemetry),
);
}
// The first Data frame records the live streaming transition. Later
// telemetry frames only refine the terminal accumulator; persisting
// every elapsed-time update would create one usage task per frame.
usage_stream_telemetry = Some(usage_frame_telemetry);
}
telemetry = Some(frame_telemetry);
apps/aether-gateway/src/main.rs
+9
View File
@@ -817,6 +817,13 @@ struct GatewayUsageArgs {
)]
queue_reclaim_interval_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_USAGE_TERMINAL_SUBMISSION_MAX_IN_FLIGHT",
default_value_t = 1_024
)]
terminal_submission_max_in_flight: u64,
#[arg(
long,
env = "AETHER_GATEWAY_USAGE_TERMINAL_ENQUEUE_MAX_IN_FLIGHT",
@@ -979,6 +986,7 @@ impl GatewayUsageArgs {
reclaim_idle_ms: self.queue_reclaim_idle_ms.max(1),
reclaim_count: self.queue_reclaim_count.max(1),
reclaim_interval_ms: self.queue_reclaim_interval_ms.max(1),
terminal_submission_max_in_flight: self.terminal_submission_max_in_flight.max(1),
terminal_enqueue_max_in_flight: self.terminal_enqueue_max_in_flight.max(1),
lifecycle_enqueue_max_in_flight: self.lifecycle_enqueue_max_in_flight.max(1),
lifecycle_enqueue_delay_ms: self.lifecycle_enqueue_delay_ms,
@@ -2489,6 +2497,7 @@ mod tests {
queue_reclaim_idle_ms: 60_000,
queue_reclaim_count: 128,
queue_reclaim_interval_ms: 5_000,
terminal_submission_max_in_flight: 1_024,
terminal_enqueue_max_in_flight: 1_024,
lifecycle_enqueue_max_in_flight: 512,
lifecycle_enqueue_delay_ms: 1_000,
apps/aether-gateway/src/state/core.rs
+74 -2
View File
@@ -2975,6 +2975,30 @@ fn usage_runtime_metric_samples(
MetricKind::Gauge,
u64::from(snapshot.retry_deferred_lifecycle_events),
),
MetricSample::new(
"usage_runtime_terminal_submission_limit",
"Maximum concurrent end-to-end terminal usage submissions.",
MetricKind::Gauge,
snapshot.terminal_submission_limit as u64,
),
MetricSample::new(
"usage_runtime_terminal_submission_in_flight",
"Current end-to-end terminal usage submissions in flight.",
MetricKind::Gauge,
snapshot.terminal_submission_in_flight as u64,
),
MetricSample::new(
"usage_runtime_terminal_submission_max_in_flight",
"Maximum observed concurrent end-to-end terminal usage submissions.",
MetricKind::Gauge,
snapshot.terminal_submission_max_in_flight as u64,
),
MetricSample::new(
"usage_runtime_terminal_submission_rejected_total",
"Total terminal usage submissions rejected by bounded ingress admission.",
MetricKind::Counter,
snapshot.terminal_submission_rejected_total,
),
MetricSample::new(
"usage_runtime_terminal_enqueue_in_flight",
"Current terminal usage enqueue operations in flight.",
@@ -2983,10 +3007,22 @@ fn usage_runtime_metric_samples(
),
MetricSample::new(
"usage_runtime_terminal_enqueue_deferred_total",
"Total terminal usage enqueue operations deferred by circuit or in-flight limits.",
"Total terminal usage enqueue operations deferred after Redis failure, circuit open, or in-flight saturation.",
MetricKind::Counter,
snapshot.terminal_enqueue_deferred_total,
),
MetricSample::new(
"usage_runtime_terminal_enqueue_deferred_direct_write_total",
"Total deferred terminal usage events persisted through bounded direct database fallback.",
MetricKind::Counter,
snapshot.terminal_enqueue_deferred_direct_write_total,
),
MetricSample::new(
"usage_runtime_terminal_enqueue_deferred_dropped_total",
"Total deferred terminal usage events dropped after all bounded fallback capacity was exhausted.",
MetricKind::Counter,
snapshot.terminal_enqueue_deferred_dropped_total,
),
MetricSample::new(
"usage_runtime_terminal_enqueue_deferred_retry_total",
"Total deferred terminal usage events scheduled for local retry.",
@@ -2999,6 +3035,42 @@ fn usage_runtime_metric_samples(
MetricKind::Counter,
snapshot.terminal_enqueue_failed_total,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_limit",
"Maximum concurrent terminal direct database fallback operations.",
MetricKind::Gauge,
snapshot.terminal_direct_fallback_limit as u64,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_in_flight",
"Current terminal direct database fallback operations in flight.",
MetricKind::Gauge,
snapshot.terminal_direct_fallback_in_flight as u64,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_max_in_flight",
"Maximum observed concurrent terminal direct database fallback operations.",
MetricKind::Gauge,
snapshot.terminal_direct_fallback_max_in_flight as u64,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_succeeded_total",
"Total terminal events persisted through direct database fallback.",
MetricKind::Counter,
snapshot.terminal_direct_fallback_succeeded_total,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_failed_total",
"Total terminal direct database fallback operations that failed.",
MetricKind::Counter,
snapshot.terminal_direct_fallback_failed_total,
),
MetricSample::new(
"usage_runtime_terminal_direct_fallback_rejected_total",
"Total terminal direct database fallback operations rejected because the writer was unavailable, pressured, or saturated.",
MetricKind::Counter,
snapshot.terminal_direct_fallback_rejected_total,
),
MetricSample::new(
"usage_runtime_lifecycle_enqueue_in_flight",
"Current lifecycle usage enqueue operations in flight.",
@@ -3055,7 +3127,7 @@ fn usage_runtime_metric_samples(
),
MetricSample::new(
"usage_runtime_enqueue_retry_closed_or_unavailable_total",
"Total usage events rejected because the local enqueue dispatcher was closed or unavailable.",
"Total usage events rejected because the local enqueue dispatcher was full, closed, or unavailable.",
MetricKind::Counter,
snapshot.enqueue_retry_closed_or_unavailable_total,
),