i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-11 19:59:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(security): harden gateway boundaries and usage policies

Browse Source 
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
elky
2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/handlers/proxy/websocket/responses/quota.rs
+56 -15
View File
@@ -18,7 +18,9 @@ use super::request::{
};
use super::state::BoundResponsesConnection;
use super::turn::{prepare_responses_websocket_turn_decision, ResponsesWebSocketTurnOutcome};
use super::upstream::{bind_responses_upstream, close_bound_upstream};
use super::upstream::{
bind_responses_upstream, close_bound_upstream, ResponsesWebSocketUpstreamBindError,
};
use crate::clock::current_unix_secs;
use crate::handlers::proxy::websocket::ingress::WebSocketRequestContext;
use crate::handlers::proxy::websocket::session::WEBSOCKET_LOG_TRANSPORT;
@@ -103,7 +105,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
state: &AppState,
context: &WebSocketRequestContext,
_previous_settled: PreviousAttemptSettled,
) -> bool {
) -> Result<bool, ()> {
// `LogicalTurn::client_event` is intentionally redacted before it is
// retained for replay. The binding, however, keeps the hash of the raw
// client-side Responses Lite tools/instructions so a later continuation
@@ -113,7 +115,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
// redacted replay event.
let responses_lite_static_config = bound.responses_lite_static_config.clone();
let Some(active) = bound.turn_state.logical_mut() else {
return false;
return Ok(false);
};
if let Some(reason) = active.quota_retry_block_reason() {
debug!(
@@ -128,7 +130,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
reason,
"gateway will not transparently replay an unsafe Responses WebSocket turn"
);
return false;
return Ok(false);
}
active.retry_attempted = true;
active.turn_attempt = active.turn_attempt.saturating_add(1);
@@ -142,12 +144,14 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
trace_id = %context.trace_id,
"gateway refused to retry a WebSocket turn without its live authorization snapshot"
);
return false;
return Ok(false);
};
let turn_index = active.turn_index;
let logical_turn_id = active.logical_turn_id.clone();
let codex_fingerprint_context = active.codex_fingerprint_context.clone();
let turn_attempt = active.turn_attempt;
let plan_usage_permit = active.plan_usage_permit.clone();
let plan_usage_policy_snapshot = active.plan_usage_policy_snapshot.clone();
let retry_exclusion_until_unix_secs = bound
.pending_adapter_drain
@@ -193,7 +197,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
exhausted_key_id = ?exhausted_key_id,
"gateway could not find an alternate Responses WebSocket provider after quota exhaustion"
);
return false;
return Ok(false);
}
Err(error) => {
warn!(
@@ -206,7 +210,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
error = ?error,
"gateway could not plan an alternate Responses WebSocket provider after quota exhaustion"
);
return false;
return Ok(false);
}
};
let OwnedResponsesWebSocketDecision {
@@ -228,7 +232,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
key_id = ?decision.key_id,
"gateway rejected an alternate Responses WebSocket plan that reused the exhausted key"
);
return false;
return Ok(false);
}
let provider_event = match planned_response_create_event(
&decision,
@@ -250,7 +254,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
error_code = code,
"gateway could not rebuild a Responses response.create for transparent quota retry"
);
return false;
return Ok(false);
}
};
let replacement_provider_store = provider_event.get("store") == Some(&Value::Bool(true));
@@ -272,6 +276,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
&turn_control.decision,
turn_decision,
&client_event,
plan_usage_policy_snapshot,
planned_lease,
)
.await
@@ -287,7 +292,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
error = ?error,
"gateway could not start usage and audit tracking for transparent quota retry"
);
return false;
return Ok(false);
}
};
let mut replacement = match bind_responses_upstream(
@@ -295,11 +300,41 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
normalization,
&client_event,
adapter,
plan_usage_permit.as_ref(),
|state| turn.record_upstream_request_state(state),
)
.await
{
Ok(connection) => connection,
Err(code) => {
Err(ResponsesWebSocketUpstreamBindError::PlanUsagePermitLost) => {
turn.release_plan_usage_cost_before_upstream_send(
state,
"transparent_retry_plan_usage_permit_lost",
)
.await;
queue_turn_finalization(
bound,
state,
turn,
ResponsesWebSocketTurnOutcome::connection_admission_lost(),
)
.await;
warn!(
event_name = "responses_websocket_quota_retry_plan_usage_concurrency_lost",
log_type = "ops",
transport = WEBSOCKET_LOG_TRANSPORT,
websocket = true,
trace_id = %context.trace_id,
"gateway stopped a transparent Responses WebSocket retry before its upstream send after the subscription plan concurrency lease became unhealthy"
);
return Err(());
}
Err(ResponsesWebSocketUpstreamBindError::Transport(code)) => {
turn.release_plan_usage_cost_before_upstream_send(
state,
"transparent_retry_rebind_failed",
)
.await;
queue_turn_finalization(
bound,
state,
@@ -316,11 +351,10 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
error_code = code,
"gateway could not bind an alternate Responses WebSocket provider after quota exhaustion"
);
return false;
return Ok(false);
}
};
turn.mark_upstream_request_sent();
turn.set_provider_response_headers(replacement.upstream_response_headers.clone());
let replacement_upstream = replacement
.upstream
@@ -348,8 +382,15 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
// drop guard 结算并让调用方走「透明重试失败」分支，不静默丢弃一条已经写了
// pending usage 行、占着 candidate 和 pool key lease 的 attempt。
if let Err(orphan) = bound.turn_state.resume(turn) {
let mut orphan = orphan;
orphan
.release_plan_usage_cost_before_upstream_send(
state,
"transparent_retry_state_handoff_failed",
)
.await;
drop(orphan);
return false;
return Ok(false);
}
if let Some(logical) = bound.turn_state.logical_mut() {
logical.provider_store = replacement_provider_store;
@@ -369,7 +410,7 @@ pub(super) async fn retry_active_turn_after_quota_exhaustion(
key_id = ?bound.decision_template.key_id,
"gateway transparently rebound a Responses WebSocket turn after quota exhaustion"
);
true
Ok(true)
}
fn responses_lite_static_config_after_rebind(