mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-12 06:00:20 +08:00
修复:Kiro 模拟缓存接入共享运行时
This commit is contained in:
@@ -1,9 +1,13 @@
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::fmt::Write as _;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_runtime_state::{DataLayerError, RuntimeState};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::warn;
|
||||
|
||||
const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(300);
|
||||
const ONE_HOUR_CACHE_TTL: Duration = Duration::from_secs(3600);
|
||||
@@ -38,6 +42,12 @@ struct KiroPromptCacheEntry {
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
|
||||
struct KiroPromptCacheRuntimeEntry {
|
||||
token_count: u64,
|
||||
ttl_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub(crate) struct KiroPromptCacheUsage {
|
||||
pub(crate) cache_creation_input_tokens: u64,
|
||||
@@ -66,6 +76,153 @@ pub(crate) fn kiro_prompt_cache_tracker() -> &'static KiroPromptCacheTracker {
|
||||
KIRO_PROMPT_CACHE_TRACKER.get_or_init(KiroPromptCacheTracker::default)
|
||||
}
|
||||
|
||||
pub(crate) async fn compute_kiro_prompt_cache_usage(
|
||||
runtime_state: &RuntimeState,
|
||||
credential_id: String,
|
||||
profile: &KiroPromptCacheProfile,
|
||||
) -> KiroPromptCacheUsage {
|
||||
match compute_kiro_prompt_cache_usage_with_runtime_state(
|
||||
runtime_state,
|
||||
credential_id.as_str(),
|
||||
profile,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(usage) => usage,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = "kiro_simulated_cache_runtime_state_failed",
|
||||
log_type = "event",
|
||||
error = ?err,
|
||||
"failed to update Kiro simulated cache runtime state; falling back to process-local tracker"
|
||||
);
|
||||
kiro_prompt_cache_tracker().compute_and_update(credential_id, profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn compute_kiro_prompt_cache_usage_with_runtime_state(
|
||||
runtime_state: &RuntimeState,
|
||||
credential_id: &str,
|
||||
profile: &KiroPromptCacheProfile,
|
||||
) -> Result<KiroPromptCacheUsage, DataLayerError> {
|
||||
let last_breakpoint = match profile.breakpoints.last().copied() {
|
||||
Some(last_breakpoint) => last_breakpoint,
|
||||
None => return Ok(KiroPromptCacheUsage::default()),
|
||||
};
|
||||
|
||||
let reversed_candidates = profile
|
||||
.match_candidates
|
||||
.iter()
|
||||
.rev()
|
||||
.copied()
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_keys = reversed_candidates
|
||||
.iter()
|
||||
.map(|candidate| kiro_prompt_cache_runtime_key(credential_id, &candidate.fingerprint))
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_values = runtime_state.kv_get_many(&candidate_keys).await?;
|
||||
let mut existing_entries = HashMap::<String, KiroPromptCacheRuntimeEntry>::new();
|
||||
let mut matched_tokens = 0u64;
|
||||
let mut matched_refresh: Option<(String, KiroPromptCacheRuntimeEntry)> = None;
|
||||
|
||||
for ((candidate, key), value) in reversed_candidates
|
||||
.iter()
|
||||
.zip(candidate_keys.iter())
|
||||
.zip(candidate_values)
|
||||
{
|
||||
let Some(entry) = value
|
||||
.as_deref()
|
||||
.and_then(parse_kiro_prompt_cache_runtime_entry)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
existing_entries.insert(key.clone(), entry);
|
||||
if matched_tokens == 0 {
|
||||
matched_tokens = entry
|
||||
.token_count
|
||||
.min(candidate.cumulative_tokens)
|
||||
.min(profile.total_input_tokens);
|
||||
matched_refresh = Some((key.clone(), entry));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((key, entry)) = matched_refresh {
|
||||
runtime_state
|
||||
.kv_set(
|
||||
key.as_str(),
|
||||
encode_kiro_prompt_cache_runtime_entry(entry),
|
||||
Some(Duration::from_secs(entry.ttl_secs.max(1))),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let creation_tokens = last_breakpoint
|
||||
.cumulative_tokens
|
||||
.min(profile.total_input_tokens)
|
||||
.saturating_sub(matched_tokens);
|
||||
|
||||
for breakpoint in &profile.breakpoints {
|
||||
let key = kiro_prompt_cache_runtime_key(credential_id, &breakpoint.fingerprint);
|
||||
let ttl_secs = breakpoint.ttl.as_secs().max(1);
|
||||
let entry = existing_entries
|
||||
.get(&key)
|
||||
.copied()
|
||||
.map(|existing| KiroPromptCacheRuntimeEntry {
|
||||
token_count: existing.token_count.max(breakpoint.cumulative_tokens),
|
||||
ttl_secs: existing.ttl_secs.max(ttl_secs),
|
||||
})
|
||||
.unwrap_or(KiroPromptCacheRuntimeEntry {
|
||||
token_count: breakpoint.cumulative_tokens,
|
||||
ttl_secs,
|
||||
});
|
||||
runtime_state
|
||||
.kv_set(
|
||||
key.as_str(),
|
||||
encode_kiro_prompt_cache_runtime_entry(entry),
|
||||
Some(Duration::from_secs(entry.ttl_secs.max(1))),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(KiroPromptCacheUsage {
|
||||
cache_creation_input_tokens: creation_tokens,
|
||||
cache_read_input_tokens: matched_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_kiro_prompt_cache_runtime_entry(value: &str) -> Option<KiroPromptCacheRuntimeEntry> {
|
||||
serde_json::from_str::<KiroPromptCacheRuntimeEntry>(value)
|
||||
.ok()
|
||||
.filter(|entry| entry.token_count > 0 && entry.ttl_secs > 0)
|
||||
}
|
||||
|
||||
fn encode_kiro_prompt_cache_runtime_entry(entry: KiroPromptCacheRuntimeEntry) -> String {
|
||||
serde_json::to_string(&entry).unwrap_or_else(|_| {
|
||||
format!(
|
||||
r#"{{"token_count":{},"ttl_secs":{}}}"#,
|
||||
entry.token_count, entry.ttl_secs
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn kiro_prompt_cache_runtime_key(credential_id: &str, fingerprint: &[u8; 32]) -> String {
|
||||
let credential_hash: [u8; 32] = Sha256::digest(credential_id.as_bytes()).into();
|
||||
format!(
|
||||
"kiro:prompt-cache:{}:{}",
|
||||
hex_digest(&credential_hash),
|
||||
hex_digest(fingerprint)
|
||||
)
|
||||
}
|
||||
|
||||
fn hex_digest(bytes: &[u8]) -> String {
|
||||
let mut output = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
let _ = write!(&mut output, "{byte:02x}");
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
pub(crate) fn build_kiro_prompt_cache_profile(
|
||||
request_body: &Value,
|
||||
total_input_tokens: u64,
|
||||
@@ -705,6 +862,7 @@ impl KiroPromptCacheTracker {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aether_runtime_state::MemoryRuntimeStateConfig;
|
||||
|
||||
fn long_text(label: &str) -> String {
|
||||
format!("{} {}", label, "cacheable prompt chunk ".repeat(300))
|
||||
@@ -802,6 +960,33 @@ mod tests {
|
||||
assert!(profile.breakpoints[0].cumulative_tokens < profile.total_input_tokens);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_state_tracker_reads_cached_prefix_across_calls() {
|
||||
let request = serde_json::json!({
|
||||
"model": "claude-sonnet-4.6",
|
||||
"system": [{
|
||||
"type": "text",
|
||||
"text": long_text("runtime shared system"),
|
||||
"cache_control": {"type": "ephemeral"}
|
||||
}],
|
||||
"messages": [{"role": "user", "content": "reuse runtime cache"}]
|
||||
});
|
||||
let profile =
|
||||
build_kiro_prompt_cache_profile(&request, estimate_kiro_prompt_input_tokens(&request))
|
||||
.expect("cacheable request should create a cache profile");
|
||||
let runtime = RuntimeState::memory(MemoryRuntimeStateConfig::default());
|
||||
|
||||
let first =
|
||||
compute_kiro_prompt_cache_usage(&runtime, "runtime-cred".to_string(), &profile).await;
|
||||
let second =
|
||||
compute_kiro_prompt_cache_usage(&runtime, "runtime-cred".to_string(), &profile).await;
|
||||
|
||||
assert!(first.cache_creation_input_tokens > 0);
|
||||
assert_eq!(first.cache_read_input_tokens, 0);
|
||||
assert_eq!(second.cache_creation_input_tokens, 0);
|
||||
assert!(second.cache_read_input_tokens > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracker_refreshes_cached_prefix_ttl_on_read() {
|
||||
let base = serde_json::json!({
|
||||
|
||||
@@ -15,8 +15,8 @@ use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::execution_runtime::kiro_cache::{
|
||||
billed_input_tokens, build_kiro_prompt_cache_profile, estimate_kiro_prompt_input_tokens,
|
||||
kiro_prompt_cache_tracker, kiro_simulated_cache_enabled_from_provider_config,
|
||||
billed_input_tokens, build_kiro_prompt_cache_profile, compute_kiro_prompt_cache_usage,
|
||||
estimate_kiro_prompt_input_tokens, kiro_simulated_cache_enabled_from_provider_config,
|
||||
KiroPromptCacheProfile, KiroPromptCacheUsage,
|
||||
};
|
||||
use crate::execution_runtime::ndjson::encode_stream_frame_ndjson;
|
||||
@@ -160,14 +160,17 @@ pub(crate) async fn maybe_execute_kiro_web_search_stream(
|
||||
|
||||
let search_results = parse_mcp_search_results(&mcp_execution.result);
|
||||
let cache_usage = if kiro_simulated_cache_enabled(state, plan).await {
|
||||
request
|
||||
.cache_profile
|
||||
.as_ref()
|
||||
.map(|profile| {
|
||||
kiro_prompt_cache_tracker()
|
||||
.compute_and_update(kiro_cache_credential_id(plan), profile)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
match request.cache_profile.as_ref() {
|
||||
Some(profile) => {
|
||||
compute_kiro_prompt_cache_usage(
|
||||
state.runtime_state(),
|
||||
kiro_cache_credential_id(plan),
|
||||
profile,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => KiroPromptCacheUsage::default(),
|
||||
}
|
||||
} else {
|
||||
KiroPromptCacheUsage::default()
|
||||
};
|
||||
|
||||
@@ -65,7 +65,7 @@ use crate::execution_runtime::chatgpt_web_image::maybe_execute_chatgpt_web_image
|
||||
use crate::execution_runtime::grok::maybe_execute_grok_stream;
|
||||
use crate::execution_runtime::kiro_cache::{
|
||||
billed_input_tokens as kiro_billed_input_tokens, build_kiro_prompt_cache_profile,
|
||||
estimate_kiro_prompt_input_tokens, kiro_prompt_cache_tracker,
|
||||
compute_kiro_prompt_cache_usage, estimate_kiro_prompt_input_tokens, kiro_prompt_cache_tracker,
|
||||
kiro_simulated_cache_enabled_from_provider_config,
|
||||
kiro_simulated_cache_enabled_from_report_context, KiroPromptCacheUsage,
|
||||
KIRO_SIMULATED_CACHE_ENABLED_CONTEXT_FIELD,
|
||||
@@ -402,7 +402,8 @@ async fn seed_kiro_simulated_cache_enabled(
|
||||
}
|
||||
}
|
||||
|
||||
fn seed_kiro_report_context_prompt_cache_usage(
|
||||
async fn seed_kiro_report_context_prompt_cache_usage(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: &mut Option<Value>,
|
||||
) {
|
||||
@@ -450,8 +451,12 @@ fn seed_kiro_report_context_prompt_cache_usage(
|
||||
return;
|
||||
};
|
||||
|
||||
let cache_usage = kiro_prompt_cache_tracker()
|
||||
.compute_and_update(kiro_stream_cache_credential_id(plan), &profile);
|
||||
let cache_usage = compute_kiro_prompt_cache_usage(
|
||||
state.runtime_state(),
|
||||
kiro_stream_cache_credential_id(plan),
|
||||
&profile,
|
||||
)
|
||||
.await;
|
||||
if cache_usage.cache_creation_input_tokens == 0 && cache_usage.cache_read_input_tokens == 0 {
|
||||
return;
|
||||
}
|
||||
@@ -1984,7 +1989,7 @@ async fn execute_stream_from_frame_stream(
|
||||
seed_kiro_report_context_input_tokens(&plan, &mut report_context);
|
||||
if status_code == 200 {
|
||||
seed_kiro_simulated_cache_enabled(state, &plan, &mut report_context).await;
|
||||
seed_kiro_report_context_prompt_cache_usage(&plan, &mut report_context);
|
||||
seed_kiro_report_context_prompt_cache_usage(state, &plan, &mut report_context).await;
|
||||
}
|
||||
let mut buffered_frames = VecDeque::new();
|
||||
let mut stream_terminal_summary: Option<ExecutionStreamTerminalSummary> = None;
|
||||
@@ -4741,9 +4746,11 @@ mod tests {
|
||||
"original_request_body": request_body,
|
||||
"kiro_simulated_cache_enabled": true,
|
||||
}));
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
super::seed_kiro_report_context_input_tokens(&plan, &mut report_context);
|
||||
super::seed_kiro_report_context_prompt_cache_usage(&plan, &mut report_context);
|
||||
super::seed_kiro_report_context_prompt_cache_usage(&state, &plan, &mut report_context)
|
||||
.await;
|
||||
|
||||
let context = report_context.as_ref().expect("context should exist");
|
||||
assert!(context
|
||||
@@ -4810,9 +4817,11 @@ mod tests {
|
||||
let mut report_context = Some(json!({
|
||||
"original_request_body": request_body,
|
||||
}));
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
super::seed_kiro_report_context_input_tokens(&plan, &mut report_context);
|
||||
super::seed_kiro_report_context_prompt_cache_usage(&plan, &mut report_context);
|
||||
super::seed_kiro_report_context_prompt_cache_usage(&state, &plan, &mut report_context)
|
||||
.await;
|
||||
|
||||
let context = report_context.as_ref().expect("context should exist");
|
||||
assert!(context
|
||||
|
||||
@@ -41,6 +41,12 @@ use crate::clock::current_unix_ms as current_request_candidate_unix_ms;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::execution_runtime::chatgpt_web_image::maybe_execute_chatgpt_web_image_sync;
|
||||
use crate::execution_runtime::grok::maybe_execute_grok_sync;
|
||||
use crate::execution_runtime::kiro_cache::{
|
||||
build_kiro_prompt_cache_profile, compute_kiro_prompt_cache_usage,
|
||||
estimate_kiro_prompt_input_tokens, kiro_simulated_cache_enabled_from_provider_config,
|
||||
kiro_simulated_cache_enabled_from_report_context, KiroPromptCacheUsage,
|
||||
KIRO_SIMULATED_CACHE_ENABLED_CONTEXT_FIELD,
|
||||
};
|
||||
use crate::execution_runtime::oauth_retry::refresh_oauth_plan_auth_for_retry;
|
||||
#[cfg(test)]
|
||||
use crate::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_runtime;
|
||||
@@ -355,6 +361,180 @@ fn build_sync_report_payload(
|
||||
}
|
||||
}
|
||||
|
||||
fn seed_kiro_sync_report_context_input_tokens(
|
||||
plan: &ExecutionPlan,
|
||||
report_context: &mut Option<Value>,
|
||||
) {
|
||||
if !plan
|
||||
.provider_name
|
||||
.as_deref()
|
||||
.is_some_and(|provider_name| provider_name.eq_ignore_ascii_case("Kiro"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(context) = report_context.as_mut().and_then(Value::as_object_mut) else {
|
||||
return;
|
||||
};
|
||||
if context
|
||||
.get("input_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.is_some_and(|input_tokens| input_tokens > 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(original_request_body) = context.get("original_request_body").cloned() else {
|
||||
return;
|
||||
};
|
||||
let estimated_input_tokens = estimate_kiro_prompt_input_tokens(&original_request_body);
|
||||
context.insert(
|
||||
"input_tokens".to_string(),
|
||||
Value::from(estimated_input_tokens),
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_kiro_sync_simulated_cache_enabled(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: &mut Option<Value>,
|
||||
) {
|
||||
if !plan
|
||||
.provider_name
|
||||
.as_deref()
|
||||
.is_some_and(|provider_name| provider_name.eq_ignore_ascii_case("Kiro"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let enabled = match state
|
||||
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&plan.provider_id))
|
||||
.await
|
||||
{
|
||||
Ok(providers) => providers
|
||||
.iter()
|
||||
.find(|provider| provider.id == plan.provider_id)
|
||||
.filter(|provider| provider.provider_type.eq_ignore_ascii_case("kiro"))
|
||||
.is_some_and(|provider| {
|
||||
kiro_simulated_cache_enabled_from_provider_config(provider.config.as_ref())
|
||||
}),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = "kiro_simulated_cache_config_read_failed",
|
||||
log_type = "event",
|
||||
request_id = %plan.request_id,
|
||||
provider_id = %plan.provider_id,
|
||||
error = ?err,
|
||||
"failed to read Kiro simulated cache provider config; defaulting disabled"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
let Some(context) = report_context.as_mut().and_then(Value::as_object_mut) else {
|
||||
return;
|
||||
};
|
||||
if enabled {
|
||||
context.insert(
|
||||
KIRO_SIMULATED_CACHE_ENABLED_CONTEXT_FIELD.to_string(),
|
||||
Value::Bool(true),
|
||||
);
|
||||
} else {
|
||||
context.remove(KIRO_SIMULATED_CACHE_ENABLED_CONTEXT_FIELD);
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_kiro_sync_report_context_prompt_cache_usage(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: &mut Option<Value>,
|
||||
) {
|
||||
if !plan
|
||||
.provider_name
|
||||
.as_deref()
|
||||
.is_some_and(|provider_name| provider_name.eq_ignore_ascii_case("Kiro"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let simulated_cache_enabled =
|
||||
kiro_simulated_cache_enabled_from_report_context(report_context.as_ref());
|
||||
let Some(context) = report_context.as_mut().and_then(Value::as_object_mut) else {
|
||||
return;
|
||||
};
|
||||
if context
|
||||
.get("kiro_web_search_mcp")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if !simulated_cache_enabled {
|
||||
return;
|
||||
}
|
||||
if kiro_cache_usage_from_context_object(context).is_some() {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(original_request_body) = context.get("original_request_body").cloned() else {
|
||||
return;
|
||||
};
|
||||
let input_tokens = context
|
||||
.get("input_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or_else(|| {
|
||||
let estimated = estimate_kiro_prompt_input_tokens(&original_request_body);
|
||||
context.insert("input_tokens".to_string(), Value::from(estimated));
|
||||
estimated
|
||||
});
|
||||
let Some(profile) = build_kiro_prompt_cache_profile(&original_request_body, input_tokens)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
let cache_usage = compute_kiro_prompt_cache_usage(
|
||||
state.runtime_state(),
|
||||
kiro_sync_cache_credential_id(plan),
|
||||
&profile,
|
||||
)
|
||||
.await;
|
||||
if cache_usage.cache_creation_input_tokens == 0 && cache_usage.cache_read_input_tokens == 0 {
|
||||
return;
|
||||
}
|
||||
context.insert(
|
||||
"cache_creation_input_tokens".to_string(),
|
||||
Value::from(cache_usage.cache_creation_input_tokens),
|
||||
);
|
||||
context.insert(
|
||||
"cache_read_input_tokens".to_string(),
|
||||
Value::from(cache_usage.cache_read_input_tokens),
|
||||
);
|
||||
}
|
||||
|
||||
fn kiro_sync_cache_credential_id(plan: &ExecutionPlan) -> String {
|
||||
format!("{}:{}:{}", plan.provider_id, plan.endpoint_id, plan.key_id)
|
||||
}
|
||||
|
||||
fn kiro_cache_usage_from_context_object(
|
||||
context: &serde_json::Map<String, Value>,
|
||||
) -> Option<KiroPromptCacheUsage> {
|
||||
let cache_creation_input_tokens = context
|
||||
.get("cache_creation_input_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let cache_read_input_tokens = context
|
||||
.get("cache_read_input_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
(cache_creation_input_tokens > 0 || cache_read_input_tokens > 0).then_some(
|
||||
KiroPromptCacheUsage {
|
||||
cache_creation_input_tokens,
|
||||
cache_read_input_tokens,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn invalid_gemini_provider_success_message(
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&Value>,
|
||||
@@ -1934,8 +2114,13 @@ async fn execute_execution_runtime_sync_impl(
|
||||
}
|
||||
let status_code = result.status_code;
|
||||
let has_body_bytes = body_base64.is_some();
|
||||
let report_context =
|
||||
let mut report_context =
|
||||
attach_provider_response_headers_to_report_context(report_context, &headers);
|
||||
if (200..300).contains(&status_code) {
|
||||
seed_kiro_sync_report_context_input_tokens(&plan, &mut report_context);
|
||||
seed_kiro_sync_simulated_cache_enabled(state, &plan, &mut report_context).await;
|
||||
seed_kiro_sync_report_context_prompt_cache_usage(state, &plan, &mut report_context).await;
|
||||
}
|
||||
let mut client_headers = headers.clone();
|
||||
apply_endpoint_response_header_rules(state, &plan, &mut client_headers, body_json.as_ref())
|
||||
.await?;
|
||||
@@ -2504,6 +2689,45 @@ mod tests {
|
||||
plan
|
||||
}
|
||||
|
||||
fn test_kiro_sync_plan() -> ExecutionPlan {
|
||||
ExecutionPlan {
|
||||
request_id: "req-kiro-sync-cache-1".to_string(),
|
||||
candidate_id: Some("candidate-kiro-sync-cache-1".to_string()),
|
||||
provider_name: Some("Kiro".to_string()),
|
||||
provider_id: "provider-kiro-sync-1".to_string(),
|
||||
endpoint_id: "endpoint-kiro-sync-1".to_string(),
|
||||
key_id: "key-kiro-sync-1".to_string(),
|
||||
method: "POST".to_string(),
|
||||
url: "https://kiro.example/generateAssistantResponse".to_string(),
|
||||
headers: BTreeMap::new(),
|
||||
content_type: Some("application/json".to_string()),
|
||||
content_encoding: None,
|
||||
body: aether_contracts::RequestBody::from_json(json!({
|
||||
"model": "claude-sonnet-4",
|
||||
"messages": [{"role": "user", "content": "hello kiro"}],
|
||||
})),
|
||||
stream: false,
|
||||
client_api_format: "claude:messages".to_string(),
|
||||
provider_api_format: "claude:messages".to_string(),
|
||||
model_name: Some("claude-sonnet-4".to_string()),
|
||||
proxy: None,
|
||||
transport_profile: None,
|
||||
timeouts: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_kiro_sync_cacheable_request_body() -> serde_json::Value {
|
||||
json!({
|
||||
"model": "claude-sonnet-4",
|
||||
"system": [{
|
||||
"type": "text",
|
||||
"text": format!("sync cacheable prompt {}", "cacheable prompt chunk ".repeat(300)),
|
||||
"cache_control": {"type": "ephemeral"}
|
||||
}],
|
||||
"messages": [{"role": "user", "content": "reuse this Kiro prompt"}]
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_gemini_provider_success_uses_plan_format_when_context_is_missing() {
|
||||
let plan = test_gemini_chat_plan();
|
||||
@@ -2659,6 +2883,68 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_sync_report_context_seeds_input_tokens_from_original_request_body() {
|
||||
let plan = test_kiro_sync_plan();
|
||||
let mut report_context = Some(json!({
|
||||
"original_request_body": test_kiro_sync_cacheable_request_body(),
|
||||
}));
|
||||
|
||||
seed_kiro_sync_report_context_input_tokens(&plan, &mut report_context);
|
||||
|
||||
assert!(report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("input_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.is_some_and(|tokens| tokens > 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kiro_sync_report_context_applies_prompt_cache_usage_from_tracker() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let plan = test_kiro_sync_plan();
|
||||
|
||||
let mut first_report_context = Some(json!({
|
||||
"original_request_body": test_kiro_sync_cacheable_request_body(),
|
||||
"kiro_simulated_cache_enabled": true,
|
||||
}));
|
||||
seed_kiro_sync_report_context_input_tokens(&plan, &mut first_report_context);
|
||||
seed_kiro_sync_report_context_prompt_cache_usage(&state, &plan, &mut first_report_context)
|
||||
.await;
|
||||
let first_creation = first_report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("cache_creation_input_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or_default();
|
||||
let first_read = first_report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("cache_read_input_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or_default();
|
||||
assert!(first_creation > 0);
|
||||
assert_eq!(first_read, 0);
|
||||
|
||||
let mut second_report_context = Some(json!({
|
||||
"original_request_body": test_kiro_sync_cacheable_request_body(),
|
||||
"kiro_simulated_cache_enabled": true,
|
||||
}));
|
||||
seed_kiro_sync_report_context_input_tokens(&plan, &mut second_report_context);
|
||||
seed_kiro_sync_report_context_prompt_cache_usage(&state, &plan, &mut second_report_context)
|
||||
.await;
|
||||
let second_creation = second_report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("cache_creation_input_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or_default();
|
||||
let second_read = second_report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("cache_read_input_tokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or_default();
|
||||
assert_eq!(second_creation, 0);
|
||||
assert!(second_read > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn json_whitespace_heartbeat_stream_prefixes_final_json() {
|
||||
let (tx, rx) = mpsc::channel::<Result<Bytes, IoError>>(1);
|
||||
|
||||
@@ -8,6 +8,9 @@ use super::{
|
||||
StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderModelMapping,
|
||||
DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
||||
};
|
||||
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
||||
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageReadRepository};
|
||||
use aether_usage_runtime::UsageRuntimeConfig;
|
||||
|
||||
const KIRO_CLAUDE_CLI_SYNC_TEST_STACK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
@@ -33,6 +36,30 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_completed_usage<T>(repository: &T, request_id: &str) -> StoredRequestUsageAudit
|
||||
where
|
||||
T: UsageReadRepository + ?Sized,
|
||||
{
|
||||
let timeout = std::time::Duration::from_secs(60);
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
if let Some(usage) = repository
|
||||
.find_by_request_id(request_id)
|
||||
.await
|
||||
.expect("usage should read")
|
||||
{
|
||||
if usage.status == "completed" {
|
||||
return usage;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"usage {request_id} should complete within {timeout:?}"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candidate() {
|
||||
run_kiro_claude_cli_sync_test(
|
||||
@@ -194,7 +221,7 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
Some(serde_json::json!({"url":"http://provider-proxy.internal:8080"})),
|
||||
Some(20.0),
|
||||
None,
|
||||
None,
|
||||
Some(serde_json::json!({"kiro": {"simulated_cache_enabled": true}})),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -354,14 +381,15 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&raw_body).expect("execution runtime payload should parse");
|
||||
let trace_id = parts
|
||||
.headers
|
||||
.get(TRACE_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
*seen_execution_runtime_inner.lock().expect("mutex should lock") =
|
||||
Some(SeenExecutionRuntimeSyncRequest {
|
||||
trace_id: parts
|
||||
.headers
|
||||
.get(TRACE_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
trace_id: trace_id.clone(),
|
||||
url: payload
|
||||
.get("url")
|
||||
.and_then(|value| value.as_str())
|
||||
@@ -453,7 +481,7 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
.concat();
|
||||
|
||||
Json(json!({
|
||||
"request_id": "trace-kiro-cli-local-sync-123",
|
||||
"request_id": trace_id,
|
||||
"status_code": 200,
|
||||
"headers": {
|
||||
"content-type": "application/vnd.amazon.eventstream"
|
||||
@@ -481,6 +509,7 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
sample_candidate_row(),
|
||||
]));
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider_catalog_provider()],
|
||||
vec![sample_provider_catalog_endpoint()],
|
||||
@@ -491,34 +520,51 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
|
||||
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url.clone())
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
|
||||
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
|
||||
auth_repository,
|
||||
candidate_selection_repository,
|
||||
provider_catalog_repository,
|
||||
Arc::clone(&request_candidate_repository),
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
),
|
||||
);
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..UsageRuntimeConfig::default()
|
||||
});
|
||||
let gateway = build_router_with_state(gateway_state);
|
||||
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/messages"))
|
||||
.header(http::header::CONTENT_TYPE, "application/json")
|
||||
.header(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer sk-client-kiro-cli-local-sync",
|
||||
)
|
||||
.header(TRACE_ID_HEADER, "trace-kiro-cli-local-sync-123")
|
||||
.body(
|
||||
"{\"model\":\"claude-sonnet-4\",\"messages\":[{\"role\":\"user\",\"content\":\"hello kiro\"}],\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":64}}",
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
async fn send_kiro_request(
|
||||
gateway_url: &str,
|
||||
trace_id: &str,
|
||||
body: String,
|
||||
) -> (StatusCode, String) {
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{gateway_url}/v1/messages"))
|
||||
.header(http::header::CONTENT_TYPE, "application/json")
|
||||
.header(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer sk-client-kiro-cli-local-sync",
|
||||
)
|
||||
.header(TRACE_ID_HEADER, trace_id)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
let status = response.status();
|
||||
let response_body = response.text().await.expect("body should read");
|
||||
let status = response.status();
|
||||
let response_body = response.text().await.expect("body should read");
|
||||
(status, response_body)
|
||||
}
|
||||
|
||||
let (status, response_body) = send_kiro_request(
|
||||
&gateway_url,
|
||||
"trace-kiro-cli-local-sync-123",
|
||||
"{\"model\":\"claude-sonnet-4\",\"messages\":[{\"role\":\"user\",\"content\":\"hello kiro\"}],\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":64}}".to_string(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
status == StatusCode::OK,
|
||||
"unexpected status={status} body={response_body} decision_hits={} plan_hits={} public_hits={}",
|
||||
@@ -598,6 +644,58 @@ async fn gateway_executes_kiro_claude_cli_sync_via_local_provider_catalog_candid
|
||||
"report-sync should stay local when request candidate persistence is available"
|
||||
);
|
||||
|
||||
let cacheable_request_body = serde_json::json!({
|
||||
"model": "claude-sonnet-4",
|
||||
"system": [{
|
||||
"type": "text",
|
||||
"text": format!("sync cacheable prompt {}", "cacheable prompt chunk ".repeat(300)),
|
||||
"cache_control": {"type": "ephemeral"}
|
||||
}],
|
||||
"messages": [{"role": "user", "content": "reuse this Kiro prompt"}]
|
||||
})
|
||||
.to_string();
|
||||
let (first_cache_status, first_cache_body) = send_kiro_request(
|
||||
&gateway_url,
|
||||
"trace-kiro-cli-local-sync-cache-1",
|
||||
cacheable_request_body.clone(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
first_cache_status == StatusCode::OK,
|
||||
"unexpected first cache status={first_cache_status} body={first_cache_body}"
|
||||
);
|
||||
let first_usage = wait_for_completed_usage(
|
||||
usage_repository.as_ref(),
|
||||
"trace-kiro-cli-local-sync-cache-1",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
first_usage.cache_creation_input_tokens > 0,
|
||||
"first Kiro sync cacheable request should create simulated cache"
|
||||
);
|
||||
assert_eq!(first_usage.cache_read_input_tokens, 0);
|
||||
|
||||
let (second_cache_status, second_cache_body) = send_kiro_request(
|
||||
&gateway_url,
|
||||
"trace-kiro-cli-local-sync-cache-2",
|
||||
cacheable_request_body,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
second_cache_status == StatusCode::OK,
|
||||
"unexpected second cache status={second_cache_status} body={second_cache_body}"
|
||||
);
|
||||
let second_usage = wait_for_completed_usage(
|
||||
usage_repository.as_ref(),
|
||||
"trace-kiro-cli-local-sync-cache-2",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
second_usage.cache_read_input_tokens > 0,
|
||||
"second Kiro sync cacheable request should read simulated cache"
|
||||
);
|
||||
assert_eq!(second_usage.cache_creation_input_tokens, 0);
|
||||
|
||||
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
|
||||
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
|
||||
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
||||
|
||||
Reference in New Issue
Block a user