i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-09 18:59:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(gateway): scale request hot paths for 20k streams

Browse Source 
Shard and singleflight hot-path caches, batch and prioritize candidate and usage lifecycle persistence, and extend database and pressure-test instrumentation for 20k concurrent streams.
This commit is contained in:
elky
2026-07-22 02:11:08 +08:00
parent 7756c0913f
commit fc92c4f431
124 changed files with 36325 additions and 3217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-usage/runtime/src/request_metadata.rs
+61 -3
View File
@@ -115,6 +115,37 @@ pub(crate) fn sanitize_usage_request_metadata(value: Option<Value>) -> Option<Va
(!filtered.is_empty()).then_some(Value::Object(filtered))
}
pub(crate) fn retain_first_byte_request_metadata(value: Option<Value>) -> Option<Value> {
let Value::Object(mut metadata) = sanitize_usage_request_metadata(value)? else {
return None;
};
metadata.retain(|key, _| {
matches!(
key.as_str(),
"trace_id"
| "client_ip"
| "user_agent"
| "client_family"
| "client_requested_stream"
| "upstream_is_stream"
| "client_session_affinity"
| "api_key_is_standalone"
| "request_path"
| "request_query_string"
| "request_path_and_query"
| "requested_reasoning_effort"
| "provider_reasoning_effort"
| "provider_service_tier"
| "provider_cache_ttl_minutes"
| "model_id"
| "global_model_id"
| "global_model_name"
| "proxy"
)
});
(!metadata.is_empty()).then_some(Value::Object(metadata))
}
pub(crate) fn sanitize_usage_request_metadata_ref(value: Option<&Value>) -> Option<Value> {
let object = value.and_then(Value::as_object)?;
@@ -669,9 +700,9 @@ mod tests {
attach_provider_request_body_metadata, attach_provider_response_body_metadata,
build_usage_request_metadata_seed, merge_usage_request_metadata,
merge_usage_request_metadata_owned, refresh_provider_response_body_metadata,
sanitize_usage_request_metadata, sanitize_usage_request_metadata_ref,
MAX_USAGE_REQUEST_METADATA_BYTES, MAX_USAGE_REQUEST_METADATA_DEPTH,
MAX_USAGE_REQUEST_METADATA_NODES,
retain_first_byte_request_metadata, sanitize_usage_request_metadata,
sanitize_usage_request_metadata_ref, MAX_USAGE_REQUEST_METADATA_BYTES,
MAX_USAGE_REQUEST_METADATA_DEPTH, MAX_USAGE_REQUEST_METADATA_NODES,
};
fn sample_plan() -> ExecutionPlan {
@@ -810,6 +841,33 @@ mod tests {
);
}
#[test]
fn first_byte_metadata_keeps_trace_context_and_drops_large_snapshots() {
let metadata = retain_first_byte_request_metadata(Some(json!({
"trace_id": "trace-first-byte",
"client_ip": "203.0.113.8",
"request_path": "/v1/chat/completions",
"upstream_is_stream": true,
"proxy": {"mode": "manual", "node_id": "proxy-1"},
"billing_snapshot": {"dimensions": [1, 2, 3]},
"settlement_snapshot": {"status": "pending"},
"stage_timings_ms": {"planning": 12}
})))
.expect("first-byte metadata should remain");
assert_eq!(
metadata,
json!({
"trace_id": "trace-first-byte",
"client_ip": "203.0.113.8",
"request_path": "/v1/chat/completions",
"request_path_and_query": "/v1/chat/completions",
"upstream_is_stream": true,
"proxy": {"mode": "manual", "node_id": "proxy-1"}
})
);
}
#[test]
fn sanitizes_request_path_query_metadata() {
let metadata = sanitize_usage_request_metadata(Some(json!({