i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/pr/536'

Browse Source 
# Conflicts:
#	apps/aether-gateway/src/execution_runtime/stream/execution.rs
This commit is contained in:
fawney19
2026-05-21 23:03:14 +08:00
parent 97997685b5 eeb7995214
commit d6eb41aa78
18 changed files with 1850 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
crates/aether-ai-formats/src/formats/openai/chat/stream.rs
View File

@@ -1180,6 +1180,23 @@ impl OpenAIResponsesProviderState {
}
}
}
event_type if openai_stream_payload_is_terminal_error(&value) => {
self.finished = true;
let mut payload = value.clone();
if event_type != "response.failed"
&& event_type != "response.incomplete"
&& event_type != "error"
{
payload = openai_stream_terminal_error_body(&value).unwrap_or(payload);
if let Some(object) = payload.as_object_mut() {
object.insert(
"type".to_string(),
Value::String("response.failed".to_string()),
);
}
}
out.push(self.unknown_frame(report_context, payload));
}
"response.completed" => {
let Some(response) = value.get("response").and_then(Value::as_object) else {
return Ok(out);
@@ -1291,6 +1308,8 @@ pub struct OpenAIChatClientEmitter {
model: Option<String>,
started: bool,
finished: bool,
next_tool_call_index: usize,
tool_call_index_by_canonical: BTreeMap<usize, usize>,
}
#[derive(Clone, Default)]
@@ -1375,6 +1394,17 @@ impl OpenAIChatClientEmitter {
)
}
fn chat_tool_call_index(&mut self, canonical_index: usize) -> usize {
if let Some(index) = self.tool_call_index_by_canonical.get(&canonical_index) {
return *index;
}
let index = self.next_tool_call_index;
self.next_tool_call_index += 1;
self.tool_call_index_by_canonical
.insert(canonical_index, index);
index
}
pub fn emit(&mut self, frame: CanonicalStreamFrame) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
self.update_identity(&frame);
match frame.event {
@@ -1483,6 +1513,7 @@ impl OpenAIChatClientEmitter {
name,
} => {
let mut out = self.ensure_started()?;
let chat_index = self.chat_tool_call_index(index);
out.extend(encode_json_sse(
None,
&build_openai_chat_chunk(
@@ -1492,7 +1523,7 @@ impl OpenAIChatClientEmitter {
self.model.as_deref().unwrap_or("unknown"),
String::new(),
Some(vec![json!({
"index": index,
"index": chat_index,
"id": call_id,
"type": "function",
"function": {
@@ -1507,6 +1538,7 @@ impl OpenAIChatClientEmitter {
}
CanonicalStreamEvent::ToolCallArgumentsDelta { index, arguments } => {
let mut out = self.ensure_started()?;
let chat_index = self.chat_tool_call_index(index);
out.extend(encode_json_sse(
None,
&json!({
@@ -1519,7 +1551,7 @@ impl OpenAIChatClientEmitter {
"index": 0,
"delta": {
"tool_calls": [{
"index": index,
"index": chat_index,
"function": {
"arguments": arguments,
}
@@ -1562,6 +1594,13 @@ impl OpenAIChatClientEmitter {
)?);
Ok(out)
}
CanonicalStreamEvent::UnknownEvent(payload)
if openai_stream_terminal_error_body(&payload).is_some() =>
{
self.finished = true;
let error_body = openai_stream_terminal_error_body(&payload).unwrap_or(payload);
encode_json_sse(None, &error_body)
}
CanonicalStreamEvent::UnknownEvent(_) => Ok(Vec::new()),
CanonicalStreamEvent::Finish {
finish_reason,
@@ -2508,6 +2547,29 @@ impl OpenAIResponsesClientEmitter {
}
Ok(out)
}
CanonicalStreamEvent::UnknownEvent(payload)
if openai_stream_terminal_error_body(&payload).is_some() =>
{
self.finished = true;
let raw_event = payload.get("type").and_then(Value::as_str);
let event = raw_event
.filter(|event| {
matches!(*event, "response.failed" | "response.incomplete" | "error")
})
.unwrap_or("response.failed")
.to_string();
let mut payload = if raw_event == Some(event.as_str()) {
payload
} else {
openai_stream_terminal_error_body(&payload).unwrap_or(payload)
};
if payload.get("type").is_none() {
if let Some(object) = payload.as_object_mut() {
object.insert("type".to_string(), Value::String(event.clone()));
}
}
self.encode_response_event(event.as_str(), payload)
}
CanonicalStreamEvent::UnknownEvent(_) => Ok(Vec::new()),
CanonicalStreamEvent::Finish { usage, .. } => {
if self.finished {
@@ -2655,6 +2717,27 @@ mod tests {
parts
}
fn openai_chat_tool_call_indices(sse: &str) -> Vec<u64> {
let mut indices = Vec::new();
for payload in sse.lines().filter_map(|line| line.strip_prefix("data: ")) {
let Ok(value) = serde_json::from_str::<Value>(payload) else {
continue;
};
let Some(tool_calls) = value
.pointer("/choices/0/delta/tool_calls")
.and_then(Value::as_array)
else {
continue;
};
for tool_call in tool_calls {
if let Some(index) = tool_call.get("index").and_then(Value::as_u64) {
indices.push(index);
}
}
}
indices
}
#[test]
fn openai_chat_provider_state_emits_unknown_events_for_unrecognized_deltas() {
let mut state = OpenAIChatProviderState::default();
@@ -2713,6 +2796,40 @@ mod tests {
)));
}
#[test]
fn openai_responses_provider_state_treats_failed_event_as_terminal() {
let mut state = OpenAIResponsesProviderState::default();
let report_context = json!({});
let frames = state
.push_line(
&report_context,
data_line(json!({
"type": "response.failed",
"response": {
"id": "resp_failed_123",
"model": "gpt-5.4",
"status": "failed",
"error": {
"message": "policy failure",
"type": "invalid_request_error",
"code": "cyber_policy"
}
}
})),
)
.expect("failed response event should parse");
assert!(frames.iter().any(|frame| matches!(
frame.event,
CanonicalStreamEvent::UnknownEvent(ref payload)
if payload.get("type").and_then(Value::as_str) == Some("response.failed")
)));
assert!(state
.finish(&report_context)
.expect("terminal failure should not synthesize completion")
.is_empty());
}
#[test]
fn openai_usage_derives_missing_input_tokens_from_total() {
let usage = canonical_usage_from_openai_usage(Some(&json!({
@@ -2879,6 +2996,41 @@ mod tests {
assert_eq!(response_sequence_numbers(&sse), (1..=9).collect::<Vec<_>>());
}
#[test]
fn openai_responses_client_emitter_forwards_failed_unknown_event() {
let mut emitter = OpenAIResponsesClientEmitter::default();
let bytes = emitter
.emit(CanonicalStreamFrame {
id: "resp_failed_123".to_string(),
model: "gpt-5.4".to_string(),
event: CanonicalStreamEvent::UnknownEvent(json!({
"type": "response.failed",
"response": {
"id": "resp_failed_123",
"model": "gpt-5.4",
"status": "failed",
"error": {
"message": "policy failure",
"type": "invalid_request_error",
"code": "cyber_policy"
}
}
})),
})
.expect("failed response event should encode");
let mut all = bytes;
all.extend(
emitter
.finish()
.expect("failed stream should not synthesize completion"),
);
let sse = String::from_utf8(all).expect("sse should be utf8");
assert!(sse.contains("event: response.failed\n"));
assert!(sse.contains("\"message\":\"policy failure\""));
assert!(!sse.contains("event: response.completed\n"));
}
#[test]
fn openai_responses_client_emitter_keeps_text_item_id_stable_after_text_started() {
let mut emitter = OpenAIResponsesClientEmitter::default();
@@ -3367,6 +3519,50 @@ mod tests {
assert!(sse.contains("[Image]"));
}
#[test]
fn openai_chat_client_emitter_normalizes_sparse_tool_call_indices() {
let mut emitter = OpenAIChatClientEmitter::default();
let mut bytes = Vec::new();
for event in [
CanonicalStreamEvent::ToolCallStart {
index: 1,
call_id: "call_first".to_string(),
name: "first_tool".to_string(),
},
CanonicalStreamEvent::ToolCallArgumentsDelta {
index: 1,
arguments: "{\"first\":".to_string(),
},
CanonicalStreamEvent::ToolCallStart {
index: 3,
call_id: "call_second".to_string(),
name: "second_tool".to_string(),
},
CanonicalStreamEvent::ToolCallArgumentsDelta {
index: 3,
arguments: "{\"second\":true}".to_string(),
},
CanonicalStreamEvent::ToolCallArgumentsDelta {
index: 1,
arguments: "true}".to_string(),
},
] {
bytes.extend(
emitter
.emit(CanonicalStreamFrame {
id: "chatcmpl_sparse".to_string(),
model: "claude-opus-4-6".to_string(),
event,
})
.expect("tool event should encode"),
);
}
let sse = String::from_utf8(bytes).expect("sse should be utf8");
assert_eq!(openai_chat_tool_call_indices(&sse), vec![0, 0, 1, 1, 0]);
}
#[test]
fn openai_chat_client_emitter_emits_usage_only_final_chunk() {
let mut emitter = OpenAIChatClientEmitter::default();

536
crates/aether-ai-formats/src/formats/openai/responses/codex.rs
View File

@@ -165,14 +165,25 @@ fn inject_codex_default_variation_prompt(body_object: &mut serde_json::Map<Strin
);
}
fn build_stable_codex_prompt_cache_key(user_api_key_id: &str) -> Option<String> {
let normalized = user_api_key_id.trim();
fn build_stable_codex_prompt_cache_key_from_seed(kind: &str, seed: &str) -> Option<String> {
let normalized = seed.trim();
if normalized.is_empty() {
return None;
}
let normalized_kind = kind
.trim()
.to_ascii_lowercase()
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '_' || *ch == '-')
.collect::<String>();
let normalized_kind = if normalized_kind.is_empty() {
"seed".to_string()
} else {
normalized_kind
};
let namespace = format!(
"aether:codex:prompt-cache:{CODEX_PROMPT_CACHE_NAMESPACE_VERSION}:user:{normalized}"
"aether:codex:prompt-cache:{CODEX_PROMPT_CACHE_NAMESPACE_VERSION}:{normalized_kind}:{normalized}"
);
let mut hasher = Sha1::new();
hasher.update(UUID_NAMESPACE_OID_BYTES);
@@ -186,6 +197,266 @@ fn build_stable_codex_prompt_cache_key(user_api_key_id: &str) -> Option<String>
Some(Uuid::from_bytes(bytes).to_string())
}
fn build_stable_codex_prompt_cache_key(user_api_key_id: &str) -> Option<String> {
build_stable_codex_prompt_cache_key_from_seed("user", user_api_key_id)
}
fn extract_codex_prompt_cache_session_seed(provider_request_body: &Value) -> Option<String> {
fn non_empty_str(value: Option<&Value>) -> Option<&str> {
value
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn session_seed_from_metadata(metadata: &Value) -> Option<String> {
let object = metadata.as_object()?;
non_empty_str(object.get("session_id"))
.or_else(|| non_empty_str(object.get("sessionId")))
.or_else(|| non_empty_str(object.get("conversation_id")))
.or_else(|| non_empty_str(object.get("conversationId")))
.map(|value| format!("metadata:{value}"))
.or_else(|| {
let user_id = non_empty_str(object.get("user_id"))?;
serde_json::from_str::<Value>(user_id)
.ok()
.and_then(|decoded| {
non_empty_str(decoded.get("session_id"))
.or_else(|| non_empty_str(decoded.get("sessionId")))
.or_else(|| non_empty_str(decoded.get("conversation_id")))
.or_else(|| non_empty_str(decoded.get("conversationId")))
.map(|value| format!("metadata.user_id:{value}"))
})
})
}
let object = provider_request_body.as_object()?;
non_empty_str(object.get("session_id"))
.or_else(|| non_empty_str(object.get("sessionId")))
.or_else(|| non_empty_str(object.get("conversation_id")))
.or_else(|| non_empty_str(object.get("conversationId")))
.map(|value| format!("body:{value}"))
.or_else(|| object.get("metadata").and_then(session_seed_from_metadata))
}
fn sha256_hex(input: &[u8]) -> String {
let digest = Sha256::digest(input);
let mut output = String::with_capacity(digest.len() * 2);
for byte in digest {
let _ = write!(&mut output, "{byte:02x}");
}
output
}
fn stable_json_digest(value: &Value) -> Option<String> {
serde_json::to_vec(value)
.ok()
.map(|serialized| sha256_hex(&serialized))
}
fn compact_prompt_cache_text(value: &str) -> Option<Value> {
const MAX_PROMPT_CACHE_TEXT_CHARS: usize = 4096;
let normalized = value.trim();
if normalized.is_empty() {
return None;
}
let mut text = normalized
.chars()
.take(MAX_PROMPT_CACHE_TEXT_CHARS)
.collect::<String>();
if normalized.chars().count() > MAX_PROMPT_CACHE_TEXT_CHARS {
text.push_str("...");
}
Some(Value::String(text))
}
fn compact_prompt_cache_anchor(value: &Value) -> Value {
match value {
Value::String(text) => compact_prompt_cache_text(text).unwrap_or(Value::Null),
Value::Array(items) => Value::Array(
items
.iter()
.take(16)
.map(compact_prompt_cache_anchor)
.filter(|value| !value.is_null())
.collect(),
),
Value::Object(object) => {
let mut compacted = serde_json::Map::new();
for key in [
"type",
"role",
"id",
"name",
"description",
"text",
"input_text",
"output_text",
"content",
"call_id",
"arguments",
"output",
"parameters",
"strict",
"function",
"effort",
"summary",
] {
let Some(value) = object.get(key) else {
continue;
};
let value = compact_prompt_cache_anchor(value);
if !value.is_null() {
compacted.insert(key.to_string(), value);
}
}
Value::Object(compacted)
}
Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(),
}
}
fn compact_prompt_cache_json_anchor(value: &Value) -> Value {
match value {
Value::String(text) => compact_prompt_cache_text(text).unwrap_or(Value::Null),
Value::Array(items) => Value::Array(
items
.iter()
.take(16)
.map(compact_prompt_cache_json_anchor)
.filter(|value| !value.is_null())
.collect(),
),
Value::Object(object) => {
let mut compacted = serde_json::Map::new();
let mut keys = object.keys().collect::<Vec<_>>();
keys.sort();
for key in keys {
if key == "cache_control" {
continue;
}
let Some(value) = object.get(key) else {
continue;
};
let value = compact_prompt_cache_json_anchor(value);
if !value.is_null() {
compacted.insert(key.clone(), value);
}
}
Value::Object(compacted)
}
Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(),
}
}
fn collect_codex_prompt_cache_control_anchors(value: &Value, anchors: &mut Vec<Value>) {
const MAX_PROMPT_CACHE_CONTROL_ANCHORS: usize = 16;
if anchors.len() >= MAX_PROMPT_CACHE_CONTROL_ANCHORS {
return;
}
match value {
Value::Object(object) => {
if object.contains_key("cache_control") {
let mut anchor = object.clone();
anchor.remove("cache_control");
let anchor = compact_prompt_cache_anchor(&Value::Object(anchor));
if !anchor.is_null() {
anchors.push(anchor);
}
}
for child in object.values() {
if anchors.len() >= MAX_PROMPT_CACHE_CONTROL_ANCHORS {
break;
}
collect_codex_prompt_cache_control_anchors(child, anchors);
}
}
Value::Array(items) => {
for child in items {
if anchors.len() >= MAX_PROMPT_CACHE_CONTROL_ANCHORS {
break;
}
collect_codex_prompt_cache_control_anchors(child, anchors);
}
}
_ => {}
}
}
fn extract_codex_prompt_cache_control_seed(provider_request_body: &Value) -> Option<String> {
let mut anchors = Vec::new();
collect_codex_prompt_cache_control_anchors(provider_request_body, &mut anchors);
if anchors.is_empty() {
return None;
}
let seed = json!({
"model": provider_request_body.get("model"),
"anchors": anchors,
});
stable_json_digest(&seed).map(|digest| format!("cache_control:{digest}"))
}
fn first_responses_input_anchor(input: &Value) -> Option<Value> {
let items = input.as_array()?;
let first_user_message = items.iter().find(|item| {
item.get("type")
.and_then(Value::as_str)
.is_some_and(|value| value == "message")
&& item
.get("role")
.and_then(Value::as_str)
.is_some_and(|value| value == "user")
});
let first_item = first_user_message.or_else(|| items.first())?;
let anchor = compact_prompt_cache_anchor(first_item);
(!anchor.is_null()).then_some(anchor)
}
fn extract_codex_stable_request_prompt_cache_seed(
provider_request_body: &Value,
user_api_key_id: Option<&str>,
) -> Option<String> {
let object = provider_request_body.as_object()?;
let mut seed = serde_json::Map::new();
for key in [
"model",
"instructions",
"reasoning",
"tools",
"tool_choice",
"parallel_tool_calls",
] {
if let Some(value) = object.get(key).filter(|value| !value.is_null()) {
let value = if key == "tools" {
compact_prompt_cache_json_anchor(value)
} else {
compact_prompt_cache_anchor(value)
};
seed.insert(key.to_string(), value);
}
}
if let Some(input_anchor) = object.get("input").and_then(first_responses_input_anchor) {
seed.insert("first_input".to_string(), input_anchor);
}
if let Some(user_api_key_id) = user_api_key_id
.map(str::trim)
.filter(|value| !value.is_empty())
{
seed.insert(
"api_key_id".to_string(),
Value::String(user_api_key_id.to_string()),
);
}
if seed.len() < 2 {
return None;
}
stable_json_digest(&Value::Object(seed)).map(|digest| format!("stable_request:{digest}"))
}
fn build_short_codex_header_id(seed: &str) -> Option<String> {
let normalized = seed.trim();
if normalized.is_empty() {
@@ -261,31 +532,47 @@ fn maybe_insert_default_codex_header(
provider_request_headers.insert(header_name.to_string(), header_value.to_string());
}
fn maybe_inject_codex_prompt_cache_key(
provider_request_body: &mut Value,
fn codex_prompt_cache_key_to_insert(
provider_request_body: &Value,
provider_type: &str,
provider_api_format: &str,
user_api_key_id: Option<&str>,
) {
) -> Option<String> {
if !is_codex_openai_responses_request(provider_type, provider_api_format) {
return;
return None;
}
let Some(body_object) = provider_request_body.as_object_mut() else {
return;
};
let existing = body_object
let existing = provider_request_body
.get("prompt_cache_key")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !existing.is_empty() {
return;
return None;
}
let Some(prompt_cache_key) = user_api_key_id.and_then(build_stable_codex_prompt_cache_key)
else {
extract_codex_prompt_cache_session_seed(provider_request_body)
.and_then(|seed| build_stable_codex_prompt_cache_key_from_seed("session", &seed))
.or_else(|| {
extract_codex_prompt_cache_control_seed(provider_request_body)
.and_then(|seed| build_stable_codex_prompt_cache_key_from_seed("anchor", &seed))
})
.or_else(|| {
extract_codex_stable_request_prompt_cache_seed(provider_request_body, user_api_key_id)
.and_then(|seed| build_stable_codex_prompt_cache_key_from_seed("request", &seed))
})
.or_else(|| user_api_key_id.and_then(build_stable_codex_prompt_cache_key))
}
fn insert_codex_prompt_cache_key(
provider_request_body: &mut Value,
prompt_cache_key: Option<String>,
) {
let Some(prompt_cache_key) = prompt_cache_key else {
return;
};
let Some(body_object) = provider_request_body.as_object_mut() else {
return;
};
@@ -431,6 +718,13 @@ pub fn apply_codex_openai_responses_special_body_edits(
return;
}
let prompt_cache_key = codex_prompt_cache_key_to_insert(
provider_request_body,
provider_type,
provider_api_format,
user_api_key_id,
);
let Some(body_object) = provider_request_body.as_object_mut() else {
return;
};
@@ -479,12 +773,7 @@ pub fn apply_codex_openai_responses_special_body_edits(
inject_codex_default_variation_prompt(body_object);
}
maybe_inject_codex_prompt_cache_key(
provider_request_body,
provider_type,
provider_api_format,
user_api_key_id,
);
insert_codex_prompt_cache_key(provider_request_body, prompt_cache_key);
}
pub fn apply_codex_openai_responses_chat_body_edits(
@@ -509,6 +798,9 @@ pub fn apply_codex_openai_responses_chat_body_edits(
return;
};
ensure_codex_chat_reasoning_defaults(body_object, provider_api_format, body_rules);
if let Some(prompt_cache_key) = body_object.remove("prompt_cache_key") {
body_object.insert("prompt_cache_key".to_string(), prompt_cache_key);
}
}
pub fn apply_codex_openai_responses_special_headers(
@@ -748,6 +1040,208 @@ mod tests {
);
}
#[test]
fn codex_responses_body_edits_derive_prompt_cache_key_from_session_metadata() {
let mut body_a = json!({
"input": [{"role": "user", "content": "hello"}],
"model": "gpt-5.4",
"metadata": {
"user_id": "{\"session_id\":\"session-a\",\"device_id\":\"device-a\"}"
}
});
let mut body_b = json!({
"input": [{"role": "user", "content": "hello again"}],
"model": "gpt-5.4",
"metadata": {
"user_id": "{\"session_id\":\"session-a\",\"device_id\":\"device-b\"}"
}
});
let mut body_c = json!({
"input": [{"role": "user", "content": "hello"}],
"model": "gpt-5.4",
"metadata": {"session_id": "session-b"}
});
apply_codex_openai_responses_special_body_edits(
&mut body_a,
"codex",
"openai:responses",
None,
Some("key-123"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_b,
"codex",
"openai:responses",
None,
Some("different-key"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_c,
"codex",
"openai:responses",
None,
Some("key-123"),
);
assert_eq!(body_a["prompt_cache_key"], body_b["prompt_cache_key"]);
assert_ne!(body_a["prompt_cache_key"], body_c["prompt_cache_key"]);
assert!(body_a.get("metadata").is_none());
assert!(body_b.get("metadata").is_none());
assert!(body_c.get("metadata").is_none());
}
#[test]
fn codex_responses_body_edits_derive_prompt_cache_key_from_cache_control_anchor() {
let mut body_a = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "stable project brief",
"cache_control": {"type": "ephemeral"}
}]
}, {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "new turn A"}]
}],
"model": "gpt-5.4"
});
let mut body_b = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "stable project brief",
"cache_control": {"type": "ephemeral"}
}]
}, {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "new turn B"}]
}],
"model": "gpt-5.4"
});
let mut body_c = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "different project brief",
"cache_control": {"type": "ephemeral"}
}]
}],
"model": "gpt-5.4"
});
apply_codex_openai_responses_special_body_edits(
&mut body_a,
"codex",
"openai:responses",
None,
Some("key-a"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_b,
"codex",
"openai:responses",
None,
Some("key-b"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_c,
"codex",
"openai:responses",
None,
Some("key-a"),
);
assert_eq!(body_a["prompt_cache_key"], body_b["prompt_cache_key"]);
assert_ne!(body_a["prompt_cache_key"], body_c["prompt_cache_key"]);
}
#[test]
fn codex_responses_body_edits_derive_prompt_cache_key_from_stable_request_anchor() {
let mut body_a = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "open workspace"}]
}, {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "new turn A"}]
}],
"model": "gpt-5.4",
"instructions": "Be concise.",
"tools": [{
"type": "function",
"name": "shell",
"parameters": {"type": "object", "properties": {}}
}],
"reasoning": {"effort": "medium"}
});
let mut body_b = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "open workspace"}]
}, {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "new turn B"}]
}],
"model": "gpt-5.4",
"instructions": "Be concise.",
"tools": [{
"type": "function",
"name": "shell",
"parameters": {"type": "object", "properties": {}}
}],
"reasoning": {"effort": "medium"}
});
let mut body_c = json!({
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "open another workspace"}]
}],
"model": "gpt-5.4",
"instructions": "Be concise.",
"tools": [{"type": "function", "name": "shell"}],
"reasoning": {"effort": "medium"}
});
apply_codex_openai_responses_special_body_edits(
&mut body_a,
"codex",
"openai:responses",
None,
Some("key-a"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_b,
"codex",
"openai:responses",
None,
Some("key-a"),
);
apply_codex_openai_responses_special_body_edits(
&mut body_c,
"codex",
"openai:responses",
None,
Some("key-a"),
);
assert_eq!(body_a["prompt_cache_key"], body_b["prompt_cache_key"]);
assert_ne!(body_a["prompt_cache_key"], body_c["prompt_cache_key"]);
}
#[test]
fn compact_body_edits_strip_include_store_and_stream() {
let mut provider_request_body = json!({

101
crates/aether-ai-formats/src/formats/shared/stream_core/common.rs
View File

@@ -108,6 +108,107 @@ pub fn canonical_usage_from_openai_usage(value: Option<&Value>) -> Option<Canoni
})
}
pub fn openai_stream_payload_is_terminal_error(payload: &Value) -> bool {
let event_type = payload
.get("type")
.and_then(Value::as_str)
.unwrap_or_default();
if payload.get("error").is_some() {
return true;
}
if matches!(
event_type,
"error" | "response.failed" | "response.incomplete"
) {
return true;
}
payload
.get("response")
.and_then(Value::as_object)
.and_then(|response| response.get("status"))
.and_then(Value::as_str)
.is_some_and(|status| matches!(status, "failed" | "incomplete"))
}
pub fn openai_stream_terminal_error_body(payload: &Value) -> Option<Value> {
if !openai_stream_payload_is_terminal_error(payload) {
return None;
}
let event_type = payload
.get("type")
.and_then(Value::as_str)
.unwrap_or_default();
let response = payload.get("response").and_then(Value::as_object);
let status = response
.and_then(|response| response.get("status"))
.and_then(Value::as_str);
let raw_error = response
.and_then(|response| response.get("error"))
.or_else(|| payload.get("error"));
let mut error = raw_error
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
let message = error
.get("message")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| raw_error.and_then(Value::as_str).map(ToOwned::to_owned))
.or_else(|| {
payload
.get("message")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
.or_else(|| {
response
.and_then(|response| response.get("incomplete_details"))
.and_then(|details| details.get("reason"))
.and_then(Value::as_str)
.map(|reason| format!("Response incomplete: {reason}"))
})
.or_else(|| status.map(|status| format!("Response ended with status {status}")))
.unwrap_or_else(|| "Upstream stream ended with an error".to_string());
error
.entry("message".to_string())
.or_insert_with(|| Value::String(message));
error.entry("type".to_string()).or_insert_with(|| {
if event_type == "response.incomplete" || status == Some("incomplete") {
Value::String("incomplete".to_string())
} else {
Value::String("server_error".to_string())
}
});
if !error.contains_key("code") {
if let Some(reason) = response
.and_then(|response| response.get("incomplete_details"))
.and_then(|details| details.get("reason"))
.and_then(Value::as_str)
{
error.insert("code".to_string(), Value::String(reason.to_string()));
}
}
Some(json!({ "error": Value::Object(error) }))
}
pub fn openai_stream_terminal_error_message(payload: &Value) -> Option<String> {
openai_stream_terminal_error_body(payload)
.and_then(|body| body.get("error").cloned())
.and_then(|error| {
error
.get("message")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
}
pub fn canonical_usage_from_claude_usage(value: Option<&Value>) -> Option<CanonicalUsage> {
let usage = value?.as_object()?;
let input_tokens = usage

49
crates/aether-ai-formats/src/formats/shared/stream_core/format_matrix.rs
View File

@@ -14,7 +14,8 @@ use crate::formats::shared::error_body::{
};
use crate::formats::shared::sse::encode_json_sse;
use crate::formats::shared::stream_core::common::{
decode_json_data_line, CanonicalStreamEvent, CanonicalStreamFrame, CanonicalUsage,
decode_json_data_line, openai_stream_terminal_error_body, openai_stream_terminal_error_message,
CanonicalStreamEvent, CanonicalStreamFrame, CanonicalUsage,
};
use crate::formats::shared::AiSurfaceFinalizeError;
@@ -197,6 +198,14 @@ impl StreamingStandardTerminalObserver {
summary.model = Some(model);
}
match event {
CanonicalStreamEvent::UnknownEvent(payload)
if openai_stream_terminal_error_body(&payload).is_some() =>
{
summary.unknown_event_count = summary.unknown_event_count.saturating_add(1);
summary.observed_finish = true;
summary.finish_reason = Some("error".to_string());
summary.parser_error = openai_stream_terminal_error_message(&payload);
}
CanonicalStreamEvent::UnknownEvent(_) => {
summary.unknown_event_count = summary.unknown_event_count.saturating_add(1);
}
@@ -410,7 +419,8 @@ fn parse_provider_error(
}
fn parse_openai_error(payload: &Value) -> Option<(String, Option<String>, LocalCoreSyncErrorKind)> {
let error = payload.get("error")?.as_object()?;
let error_body = openai_stream_terminal_error_body(payload)?;
let error = error_body.get("error")?.as_object()?;
let message = error.get("message").and_then(Value::as_str)?.to_string();
let code = error
.get("code")
@@ -973,6 +983,41 @@ mod tests {
assert!(!summary.observed_finish);
}
#[test]
fn terminal_observer_marks_openai_responses_failed_event_as_terminal_error() {
let mut report_context = report_context("openai:chat", "openai:responses");
report_context["provider_stream_event_api_format"] = json!("openai:responses");
let mut observer = StreamingStandardTerminalObserver::default();
observer
.push_line(
&report_context,
data_line(json!({
"type": "response.failed",
"response": {
"id": "resp_failed_123",
"model": "gpt-5.4",
"status": "failed",
"error": {
"message": "policy failure",
"type": "invalid_request_error",
"code": "cyber_policy"
}
}
})),
)
.expect("failed event should be observed");
let summary = observer
.latest_summary()
.cloned()
.expect("summary should exist");
assert!(summary.observed_finish);
assert_eq!(summary.finish_reason.as_deref(), Some("error"));
assert_eq!(summary.parser_error.as_deref(), Some("policy failure"));
assert_eq!(summary.unknown_event_count, 1);
}
#[test]
fn terminal_observer_tracks_openai_image_stream_usage() {
let mut report_context = report_context("openai:image", "openai:chat");

16
crates/aether-data/src/repository/usage/postgres/queries/upsert_sql.sql
View File

@@ -183,23 +183,35 @@ DO UPDATE SET
total_cost_usd = CASE WHEN "usage".billing_status = 'pending' AND EXCLUDED.status IN ('completed', 'failed', 'cancelled') THEN GREATEST("usage".total_cost_usd, EXCLUDED.total_cost_usd) ELSE "usage".total_cost_usd END,
actual_total_cost_usd = CASE WHEN "usage".billing_status = 'pending' AND EXCLUDED.status IN ('completed', 'failed', 'cancelled') THEN GREATEST("usage".actual_total_cost_usd, EXCLUDED.actual_total_cost_usd) ELSE "usage".actual_total_cost_usd END,
status_code = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".status_code
WHEN "usage".status = 'streaming' AND EXCLUDED.status = 'pending' THEN "usage".status_code
WHEN EXCLUDED.status IN ('pending', 'streaming', 'completed', 'cancelled') AND EXCLUDED.status_code IS NULL THEN NULL
ELSE COALESCE(EXCLUDED.status_code, "usage".status_code)
END ELSE "usage".status_code END,
error_message = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".error_message
WHEN "usage".status = 'streaming' AND EXCLUDED.status = 'pending' THEN "usage".error_message
WHEN EXCLUDED.status IN ('pending', 'streaming', 'completed', 'cancelled') THEN EXCLUDED.error_message
ELSE COALESCE(EXCLUDED.error_message, "usage".error_message)
END ELSE "usage".error_message END,
error_category = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".error_category
WHEN "usage".status = 'streaming' AND EXCLUDED.status = 'pending' THEN "usage".error_category
WHEN EXCLUDED.status IN ('pending', 'streaming', 'completed', 'cancelled') THEN EXCLUDED.error_category
ELSE COALESCE(EXCLUDED.error_category, "usage".error_category)
END ELSE "usage".error_category END,
response_time_ms = CASE WHEN "usage".billing_status = 'pending' THEN COALESCE(EXCLUDED.response_time_ms, "usage".response_time_ms) ELSE "usage".response_time_ms END,
first_byte_time_ms = CASE WHEN "usage".billing_status = 'pending' THEN COALESCE(EXCLUDED.first_byte_time_ms, "usage".first_byte_time_ms) ELSE "usage".first_byte_time_ms END,
response_time_ms = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".response_time_ms
WHEN EXCLUDED.response_time_ms IS NULL OR EXCLUDED.response_time_ms = 0 THEN COALESCE("usage".response_time_ms, EXCLUDED.response_time_ms)
ELSE EXCLUDED.response_time_ms
END ELSE "usage".response_time_ms END,
first_byte_time_ms = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".first_byte_time_ms
WHEN EXCLUDED.first_byte_time_ms IS NULL OR EXCLUDED.first_byte_time_ms = 0 THEN COALESCE("usage".first_byte_time_ms, EXCLUDED.first_byte_time_ms)
ELSE EXCLUDED.first_byte_time_ms
END ELSE "usage".first_byte_time_ms END,
status = CASE WHEN "usage".billing_status = 'pending' THEN CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND EXCLUDED.status IN ('pending', 'streaming') THEN "usage".status
WHEN "usage".status = 'streaming' AND EXCLUDED.status = 'pending' THEN "usage".status
ELSE EXCLUDED.status
END ELSE "usage".status END,

42
crates/aether-data/src/repository/usage/sqlite.rs
View File

@@ -182,17 +182,41 @@ ON CONFLICT (request_id) DO UPDATE SET
output_price_per_1m = excluded.output_price_per_1m,
total_cost_usd = excluded.total_cost_usd,
actual_total_cost_usd = excluded.actual_total_cost_usd,
status_code = excluded.status_code,
error_message = excluded.error_message,
error_category = excluded.error_category,
response_time_ms = excluded.response_time_ms,
first_byte_time_ms = excluded.first_byte_time_ms,
status = excluded.status,
status_code = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".status_code
WHEN "usage".status = 'streaming' AND excluded.status = 'pending' THEN "usage".status_code
ELSE excluded.status_code
END,
error_message = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".error_message
WHEN "usage".status = 'streaming' AND excluded.status = 'pending' THEN "usage".error_message
ELSE excluded.error_message
END,
error_category = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".error_category
WHEN "usage".status = 'streaming' AND excluded.status = 'pending' THEN "usage".error_category
ELSE excluded.error_category
END,
response_time_ms = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".response_time_ms
WHEN excluded.response_time_ms IS NULL OR excluded.response_time_ms = 0 THEN COALESCE("usage".response_time_ms, excluded.response_time_ms)
ELSE excluded.response_time_ms
END,
first_byte_time_ms = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".first_byte_time_ms
WHEN excluded.first_byte_time_ms IS NULL OR excluded.first_byte_time_ms = 0 THEN COALESCE("usage".first_byte_time_ms, excluded.first_byte_time_ms)
ELSE excluded.first_byte_time_ms
END,
status = CASE
WHEN "usage".status IN ('completed', 'failed', 'cancelled') AND excluded.status IN ('pending', 'streaming') THEN "usage".status
WHEN "usage".status = 'streaming' AND excluded.status = 'pending' THEN "usage".status
ELSE excluded.status
END,
billing_status = excluded.billing_status,
request_metadata = excluded.request_metadata,
candidate_id = excluded.candidate_id,
candidate_index = excluded.candidate_index,
key_name = excluded.key_name,
candidate_id = COALESCE(excluded.candidate_id, "usage".candidate_id),
candidate_index = COALESCE(excluded.candidate_index, "usage".candidate_index),
key_name = COALESCE(excluded.key_name, "usage".key_name),
planner_kind = excluded.planner_kind,
route_family = excluded.route_family,
route_kind = excluded.route_kind,

2
crates/aether-scheduler-core/src/request_candidate.rs
View File

@@ -463,7 +463,7 @@ pub fn build_local_request_candidate_status_record(
.map(str::trim)
.filter(|value| !value.is_empty())?;
let metadata = parse_request_candidate_report_context(report_context)?;
let candidate_index = metadata.candidate_index?;
let candidate_index = metadata.candidate_index.unwrap_or(0);
let extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
client_api_format: metadata.client_api_format.clone(),
provider_api_format: metadata.provider_api_format.clone(),

6
crates/aether-usage-runtime/src/lib.rs
View File

@@ -27,9 +27,9 @@ pub use report::{
infer_internal_finalize_signature, is_local_ai_stream_report_kind,
is_local_ai_sync_report_kind, normalize_gemini_file_name, report_request_id,
resolve_internal_finalize_route, should_handle_local_stream_report,
should_handle_local_sync_report, sync_report_represents_failure, GatewayStreamReportRequest,
GatewaySyncReportRequest, GeminiFileMappingEntry, InternalFinalizeRoute,
GEMINI_FILE_MAPPING_TTL_SECONDS,
should_handle_local_sync_report, stream_report_represents_failure,
sync_report_represents_failure, GatewayStreamReportRequest, GatewaySyncReportRequest,
GeminiFileMappingEntry, InternalFinalizeRoute, GEMINI_FILE_MAPPING_TTL_SECONDS,
};
pub use report_context::{
build_locally_actionable_report_context_from_request_candidate,

50
crates/aether-usage-runtime/src/report.rs
View File

@@ -292,6 +292,24 @@ pub fn sync_report_represents_failure(
.is_some_and(|value| !value.is_null())
}
fn stream_terminal_summary_represents_failure(summary: &ExecutionStreamTerminalSummary) -> bool {
summary.parser_error.is_some()
|| (!summary.observed_finish
&& !summary
.standardized_usage
.as_ref()
.is_some_and(aether_contracts::StandardizedUsage::has_token_signal))
}
pub fn stream_report_represents_failure(payload: &GatewayStreamReportRequest) -> bool {
payload.status_code >= 400
|| payload.report_kind.contains("error")
|| payload
.terminal_summary
.as_ref()
.is_some_and(stream_terminal_summary_represents_failure)
}
pub fn should_handle_local_sync_report(
report_context: Option<&serde_json::Value>,
report_kind: &str,
@@ -373,6 +391,7 @@ fn content_type_starts_with(headers: &BTreeMap<String, String>, expected_prefix:
mod tests {
use std::collections::BTreeMap;
use aether_contracts::ExecutionStreamTerminalSummary;
use base64::Engine as _;
use serde_json::json;
@@ -381,7 +400,8 @@ mod tests {
infer_internal_finalize_signature, is_local_ai_stream_report_kind,
is_local_ai_sync_report_kind, normalize_gemini_file_name, report_request_id,
resolve_internal_finalize_route, should_handle_local_stream_report,
should_handle_local_sync_report, sync_report_represents_failure, GatewaySyncReportRequest,
should_handle_local_sync_report, stream_report_represents_failure,
sync_report_represents_failure, GatewayStreamReportRequest, GatewaySyncReportRequest,
GeminiFileMappingEntry, InternalFinalizeRoute,
};
@@ -416,6 +436,22 @@ mod tests {
}
}
fn sample_stream_report(report_kind: &str, status_code: u16) -> GatewayStreamReportRequest {
GatewayStreamReportRequest {
trace_id: "trace-stream-123".to_string(),
report_kind: report_kind.to_string(),
report_context: None,
status_code,
headers: BTreeMap::new(),
provider_body_base64: None,
provider_body_state: None,
client_body_base64: None,
client_body_state: None,
terminal_summary: None,
telemetry: None,
}
}
#[test]
fn classifies_local_ai_sync_report_kinds() {
assert!(is_local_ai_sync_report_kind(
@@ -481,6 +517,18 @@ mod tests {
assert!(!sync_report_represents_failure(&success_payload, None));
}
#[test]
fn detects_stream_report_failure_from_terminal_summary_error() {
let mut payload = sample_stream_report("openai_responses_stream_success", 200);
payload.terminal_summary = Some(ExecutionStreamTerminalSummary {
observed_finish: true,
parser_error: Some("policy failure".to_string()),
..ExecutionStreamTerminalSummary::default()
});
assert!(stream_report_represents_failure(&payload));
}
#[test]
fn infers_internal_finalize_signature_from_context_or_report_kind() {
let from_context = sample_sync_report_with_context(

555
crates/aether-usage-runtime/src/write.rs
View File

@@ -158,6 +158,8 @@ pub struct StreamTerminalUsagePayloadSeed {
pub client_response: Option<Value>,
pub client_response_body_state: Option<UsageBodyCaptureState>,
pub standardized_usage: Option<StandardizedUsage>,
pub observed_stream_finish: Option<bool>,
pub terminal_error_message: Option<String>,
pub capture_metadata: Option<Value>,
}
@@ -183,6 +185,7 @@ pub struct TerminalUsageSeed {
pub has_format_conversion: bool,
pub is_stream: bool,
pub status_code: u16,
pub terminal_error_message: Option<String>,
pub response_time_ms: Option<u64>,
pub first_byte_time_ms: Option<u64>,
pub request_headers: Option<Value>,
@@ -506,6 +509,7 @@ fn build_terminal_usage_event_from_seed_impl(
has_format_conversion,
is_stream,
status_code,
terminal_error_message,
response_time_ms,
first_byte_time_ms,
request_headers,
@@ -530,7 +534,8 @@ fn build_terminal_usage_event_from_seed_impl(
};
let routing = merge_routing_seed_with_metadata_owned(routing, request_metadata.as_ref());
let body_refs = merge_body_refs_seed_with_metadata_owned(body_refs, request_metadata.as_ref());
let error_message = resolve_error_message(status_code, provider_response.as_ref(), None)
let error_message = terminal_error_message
.or_else(|| resolve_error_message(status_code, provider_response.as_ref(), None))
.or_else(|| resolve_error_message(status_code, client_response.as_ref(), None));
let api_family = infer_api_family(&client_contract).map(ToOwned::to_owned);
let endpoint_kind = infer_endpoint_kind(&client_contract).map(ToOwned::to_owned);
@@ -623,10 +628,6 @@ fn build_terminal_usage_event_from_seed_impl(
apply_completed_image_usage_estimate(&mut data);
}
if matches!(event_type, UsageEventType::Cancelled) {
apply_cancelled_usage_estimate(&mut data);
}
let data = if trusted_request_metadata {
sanitize_usage_event_capture_fields_trusted(data)
} else {
@@ -779,6 +780,16 @@ pub fn build_stream_terminal_usage_payload_seed(
let provider_response_headers = context_usage_value(context, "provider_response_headers")
.or_else(|| headers_to_json(&payload.headers));
let client_response_headers = headers_to_json(&payload.headers);
let observed_stream_finish = payload
.terminal_summary
.as_ref()
.map(|summary| summary.observed_finish);
let terminal_error_message = payload
.terminal_summary
.as_ref()
.and_then(|summary| summary.parser_error.clone())
.map(|message| message.trim().to_string())
.filter(|message| !message.is_empty());
StreamTerminalUsagePayloadSeed {
report_kind: payload.report_kind.clone(),
status_code: payload.status_code,
@@ -797,6 +808,8 @@ pub fn build_stream_terminal_usage_payload_seed(
.terminal_summary
.as_ref()
.and_then(|summary| summary.standardized_usage.clone()),
observed_stream_finish,
terminal_error_message,
capture_metadata: build_payload_body_capture_metadata(
payload.provider_body_base64.as_deref(),
payload.client_body_base64.as_deref(),
@@ -853,6 +866,7 @@ pub fn build_sync_terminal_usage_seed(
has_format_conversion: context_seed.has_format_conversion,
is_stream: context_seed.is_stream,
status_code,
terminal_error_message: None,
response_time_ms,
first_byte_time_ms,
request_headers: context_seed.request_headers,
@@ -894,6 +908,8 @@ pub fn build_stream_terminal_usage_seed(
client_response,
client_response_body_state,
standardized_usage,
observed_stream_finish,
terminal_error_message,
capture_metadata,
} = payload_seed;
let standardized_usage = standardized_usage.or_else(|| {
@@ -901,7 +917,28 @@ pub fn build_stream_terminal_usage_seed(
map_usage_from_response(response, context_seed.provider_contract.as_str())
})
});
let terminal_state = infer_stream_terminal_state(report_kind.as_str(), status_code, cancelled);
let missing_observed_finish = matches!(observed_stream_finish, Some(false))
&& !standardized_usage
.as_ref()
.is_some_and(StandardizedUsage::has_token_signal);
let terminal_error_message = terminal_error_message
.or_else(|| {
provider_response_full
.as_ref()
.and_then(extract_explicit_error_message_from_json)
})
.or_else(|| {
client_response
.as_ref()
.and_then(extract_explicit_error_message_from_json)
});
let terminal_state = infer_stream_terminal_state(
report_kind.as_str(),
status_code,
cancelled,
missing_observed_finish,
terminal_error_message.is_some(),
);
TerminalUsageSeed {
terminal_state,
@@ -924,6 +961,7 @@ pub fn build_stream_terminal_usage_seed(
has_format_conversion: context_seed.has_format_conversion,
is_stream: context_seed.is_stream,
status_code,
terminal_error_message,
response_time_ms,
first_byte_time_ms,
request_headers: context_seed.request_headers,
@@ -970,10 +1008,12 @@ fn infer_stream_terminal_state(
report_kind: &str,
status_code: u16,
cancelled: bool,
missing_observed_finish: bool,
terminal_error: bool,
) -> UsageTerminalState {
if cancelled || status_code == 499 || report_kind.contains("cancel") {
UsageTerminalState::Cancelled
} else if !(200..300).contains(&status_code) {
} else if !(200..300).contains(&status_code) || missing_observed_finish || terminal_error {
UsageTerminalState::Failed
} else {
UsageTerminalState::Completed
@@ -2141,6 +2181,31 @@ fn extract_explicit_error_message_from_json(value: &Value) -> Option<String> {
.and_then(|error| error.get("message"))
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| {
value
.get("response")
.and_then(|response| response.get("error"))
.and_then(|error| error.get("message"))
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
.or_else(|| {
value
.get("response")
.and_then(|response| response.get("incomplete_details"))
.and_then(|details| details.get("reason"))
.and_then(Value::as_str)
.map(|reason| format!("Response incomplete: {reason}"))
})
.or_else(|| extract_stream_error_message_from_chunks(value))
}
fn extract_stream_error_message_from_chunks(value: &Value) -> Option<String> {
value
.get("chunks")
.and_then(Value::as_array)?
.iter()
.find_map(extract_explicit_error_message_from_json)
}
fn extract_generic_error_message_from_json(value: &Value) -> Option<String> {
@@ -2303,48 +2368,6 @@ fn extract_token_counts_from_value(value: &Value) -> Option<(u64, u64, u64)> {
}
}
fn apply_cancelled_usage_estimate(data: &mut UsageEventData) {
let provider_usage_available = data
.response_body
.as_ref()
.and_then(extract_token_counts_from_value)
.is_some();
let request_usage = data
.provider_request_body
.as_ref()
.or(data.request_body.as_ref())
.and_then(estimate_request_usage);
if positive_tokens(data.input_tokens) == 0 {
if let Some(usage) = request_usage.as_ref() {
data.input_tokens = Some(usage.input_tokens);
}
}
if !provider_usage_available {
apply_cancelled_request_cache_estimate(data, request_usage.as_ref());
}
if positive_tokens(data.output_tokens) == 0 {
if let Some(output_tokens) = data
.response_body
.as_ref()
.or(data.client_response_body.as_ref())
.and_then(estimate_response_output_tokens)
{
data.output_tokens = Some(output_tokens);
}
}
if positive_tokens(data.total_tokens) == 0 {
let total_tokens =
positive_tokens(data.input_tokens).saturating_add(positive_tokens(data.output_tokens));
if total_tokens > 0 {
data.total_tokens = Some(total_tokens);
}
}
}
fn apply_completed_image_usage_estimate(data: &mut UsageEventData) {
if !usage_event_data_is_image(data) {
return;
@@ -2369,7 +2392,7 @@ fn apply_completed_image_usage_estimate(data: &mut UsageEventData) {
data.input_tokens = Some(usage.input_tokens);
}
}
apply_cancelled_request_cache_estimate(data, request_usage.as_ref());
apply_request_cache_usage_estimate(data, request_usage.as_ref());
if positive_tokens(data.total_tokens) == 0 {
let total_tokens =
positive_tokens(data.input_tokens).saturating_add(positive_tokens(data.output_tokens));
@@ -2507,7 +2530,7 @@ fn usage_event_data_is_image(data: &UsageEventData) -> bool {
.is_some_and(|value| value.eq_ignore_ascii_case("image"))
}
fn apply_cancelled_request_cache_estimate(
fn apply_request_cache_usage_estimate(
data: &mut UsageEventData,
request_usage: Option<&EstimatedRequestUsage>,
) {
@@ -2674,280 +2697,6 @@ fn estimate_text_tokens(text: &str) -> u64 {
}
}
#[derive(Default)]
struct StreamOutputEstimate {
text: String,
saw_delta: bool,
}
impl StreamOutputEstimate {
fn push_delta(&mut self, text: &str) {
if text.is_empty() {
return;
}
self.saw_delta = true;
self.text.push_str(text);
}
fn push_done(&mut self, text: &str) {
if text.is_empty() || self.saw_delta {
return;
}
self.text.push_str(text);
}
}
fn estimate_response_output_tokens(value: &Value) -> Option<u64> {
let mut estimate = StreamOutputEstimate::default();
collect_stream_output_text(value, &mut estimate);
let tokens = estimate_text_tokens(estimate.text.as_str());
(tokens > 0).then_some(tokens)
}
fn collect_stream_output_text(value: &Value, estimate: &mut StreamOutputEstimate) {
match value {
Value::String(text) => {
for_each_sse_payload(text, |payload| {
if payload == "[DONE]" {
return;
}
if let Ok(json_body) = serde_json::from_str::<Value>(payload) {
collect_stream_output_text(&json_body, estimate);
}
});
}
Value::Array(items) => {
for item in items {
collect_stream_output_text(item, estimate);
}
}
Value::Object(object) => {
if let Some(chunks) = object.get("chunks").and_then(Value::as_array) {
for chunk in chunks {
collect_stream_output_text(chunk, estimate);
}
return;
}
collect_openai_responses_output_text(object, estimate);
collect_openai_chat_output_text(object, estimate);
collect_claude_output_text(object, estimate);
collect_gemini_output_text(object, estimate);
}
_ => {}
}
}
fn collect_openai_responses_output_text(
object: &Map<String, Value>,
estimate: &mut StreamOutputEstimate,
) {
match object
.get("type")
.and_then(Value::as_str)
.unwrap_or_default()
{
"response.output_text.delta" | "response.outtext.delta" => {
if let Some(text) = openai_delta_text(object.get("delta")) {
estimate.push_delta(text.as_str());
}
}
"response.reasoning_summary_text.delta" | "response.function_call_arguments.delta" => {
if let Some(text) = object.get("delta").and_then(Value::as_str) {
estimate.push_delta(text);
}
}
"response.output_text.done" | "response.reasoning_summary_text.done" => {
if let Some(text) = object
.get("text")
.and_then(Value::as_str)
.or_else(|| part_text(object.get("part")))
{
estimate.push_done(text);
}
}
"response.function_call_arguments.done" => {
if let Some(text) = object.get("arguments").and_then(Value::as_str) {
estimate.push_done(text);
}
}
"response.output_item.done" => {
if let Some(item) = object.get("item").and_then(Value::as_object) {
collect_openai_responses_output_item_text(item, estimate);
}
}
"response.completed" => {
if let Some(response) = object.get("response").and_then(Value::as_object) {
collect_openai_responses_completed_text(response, estimate);
}
}
_ => {}
}
}
fn collect_openai_responses_completed_text(
response: &Map<String, Value>,
estimate: &mut StreamOutputEstimate,
) {
for item in response
.get("output")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
{
collect_openai_responses_output_item_text(item, estimate);
}
}
fn collect_openai_responses_output_item_text(
item: &Map<String, Value>,
estimate: &mut StreamOutputEstimate,
) {
match item.get("type").and_then(Value::as_str).unwrap_or_default() {
"message" => {
for content in item
.get("content")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
{
if content.get("type").and_then(Value::as_str) == Some("output_text") {
if let Some(text) = content.get("text").and_then(Value::as_str) {
estimate.push_done(text);
}
}
}
}
"reasoning" => {
for summary in item
.get("summary")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
{
if let Some(text) = summary.get("text").and_then(Value::as_str) {
estimate.push_done(text);
}
}
}
"function_call" => {
if let Some(arguments) = item.get("arguments").and_then(Value::as_str) {
estimate.push_done(arguments);
}
}
_ => {}
}
}
fn collect_openai_chat_output_text(
object: &Map<String, Value>,
estimate: &mut StreamOutputEstimate,
) {
for choice in object
.get("choices")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
{
if let Some(delta) = choice.get("delta").and_then(Value::as_object) {
if let Some(content) = delta.get("content").and_then(Value::as_str) {
estimate.push_delta(content);
}
if let Some(reasoning_content) = delta.get("reasoning_content").and_then(Value::as_str)
{
estimate.push_delta(reasoning_content);
}
for tool_call in delta
.get("tool_calls")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
{
if let Some(arguments) = tool_call
.get("function")
.and_then(Value::as_object)
.and_then(|function| function.get("arguments"))
.and_then(Value::as_str)
{
estimate.push_delta(arguments);
}
}
}
}
}
fn collect_claude_output_text(object: &Map<String, Value>, estimate: &mut StreamOutputEstimate) {
if object.get("type").and_then(Value::as_str) != Some("content_block_delta") {
return;
}
let Some(delta) = object.get("delta").and_then(Value::as_object) else {
return;
};
match delta
.get("type")
.and_then(Value::as_str)
.unwrap_or_default()
{
"text_delta" => {
if let Some(text) = delta.get("text").and_then(Value::as_str) {
estimate.push_delta(text);
}
}
"thinking_delta" => {
if let Some(text) = delta.get("thinking").and_then(Value::as_str) {
estimate.push_delta(text);
}
}
"input_json_delta" => {
if let Some(text) = delta.get("partial_json").and_then(Value::as_str) {
estimate.push_delta(text);
}
}
_ => {}
}
}
fn collect_gemini_output_text(object: &Map<String, Value>, estimate: &mut StreamOutputEstimate) {
for part in object
.get("candidates")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|candidate| candidate.get("content"))
.filter_map(Value::as_object)
.filter_map(|content| content.get("parts"))
.filter_map(Value::as_array)
.flatten()
.filter_map(Value::as_object)
{
if let Some(text) = part.get("text").and_then(Value::as_str) {
estimate.push_delta(text);
}
}
}
fn openai_delta_text(value: Option<&Value>) -> Option<String> {
match value {
Some(Value::String(text)) => Some(text.clone()),
Some(Value::Object(object)) => object
.get("text")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
_ => None,
}
}
fn part_text(value: Option<&Value>) -> Option<&str> {
value
.and_then(Value::as_object)
.and_then(|part| part.get("text"))
.and_then(Value::as_str)
}
fn extract_token_counts_from_json(value: &Value) -> Option<(u64, u64, u64)> {
if let Some(usage) = value.get("usage").and_then(Value::as_object) {
let input = usage
@@ -3633,7 +3382,7 @@ mod tests {
}
#[test]
fn cancelled_stream_usage_estimates_tokens_from_request_and_partial_response() {
fn cancelled_stream_usage_does_not_estimate_tokens_from_request_or_partial_response() {
let plan = ExecutionPlan {
request_id: "req-stream-cancelled-estimated-usage-1".to_string(),
candidate_id: Some("cand-stream-cancelled-estimated-usage-1".to_string()),
@@ -3692,16 +3441,14 @@ mod tests {
.expect("usage event should build");
assert_eq!(event.event_type, UsageEventType::Cancelled);
assert!(event.data.input_tokens.unwrap_or_default() > 0);
assert_eq!(event.data.output_tokens, Some(5));
assert_eq!(
event.data.total_tokens,
Some(event.data.input_tokens.unwrap_or_default() + 5)
);
assert_eq!(event.data.input_tokens, None);
assert_eq!(event.data.output_tokens, None);
assert_eq!(event.data.total_tokens, None);
assert_eq!(event.data.cache_read_input_tokens, None);
}
#[test]
fn cancelled_stream_usage_does_not_infer_cache_read_from_prompt_cache_key() {
fn cancelled_stream_usage_does_not_infer_cache_or_token_estimates_from_prompt_cache_key() {
let request_body = json!({
"model": "gpt-5.4",
"input": "Use the cached project context and answer briefly",
@@ -3752,15 +3499,81 @@ mod tests {
let event =
build_stream_terminal_usage_event(&plan, payload.report_context.as_ref(), &payload)
.expect("usage event should build");
let input_tokens = event
.data
.input_tokens
.expect("input estimate should exist");
assert_eq!(event.event_type, UsageEventType::Cancelled);
assert_eq!(event.data.input_tokens, None);
assert_eq!(event.data.output_tokens, None);
assert_eq!(event.data.total_tokens, None);
assert_eq!(event.data.cache_read_input_tokens, None);
assert_eq!(event.data.output_tokens, Some(4));
assert_eq!(event.data.total_tokens, Some(input_tokens + 4));
}
#[test]
fn cancelled_stream_usage_preserves_terminal_summary_usage() {
let plan = ExecutionPlan {
request_id: "req-stream-cancelled-summary-usage-1".to_string(),
candidate_id: Some("cand-stream-cancelled-summary-usage-1".to_string()),
provider_name: Some("OpenAI".to_string()),
provider_id: "provider-1".to_string(),
endpoint_id: "endpoint-1".to_string(),
key_id: "key-1".to_string(),
method: "POST".to_string(),
url: "https://example.com/v1/responses".to_string(),
headers: BTreeMap::new(),
content_type: Some("application/json".to_string()),
content_encoding: None,
body: RequestBody::from_json(json!({
"model": "gpt-5.4",
"input": "This cancelled request has terminal upstream usage",
"stream": true
})),
stream: true,
client_api_format: "openai:responses".to_string(),
provider_api_format: "openai:responses".to_string(),
model_name: Some("gpt-5.4".to_string()),
proxy: None,
transport_profile: None,
timeouts: None,
};
let mut standardized_usage = StandardizedUsage::new();
standardized_usage.input_tokens = 13;
standardized_usage.output_tokens = 21;
standardized_usage.cache_creation_tokens = 2;
standardized_usage.cache_read_tokens = 3;
let payload = GatewayStreamReportRequest {
trace_id: "trace-stream-cancelled-summary-usage-1".to_string(),
report_kind: "openai_responses_stream_cancelled".to_string(),
report_context: Some(json!({
"client_api_format": "openai:responses",
"provider_api_format": "openai:responses"
})),
status_code: 499,
headers: BTreeMap::new(),
provider_body_base64: None,
provider_body_state: Some(UsageBodyCaptureState::None),
client_body_base64: None,
client_body_state: Some(UsageBodyCaptureState::None),
terminal_summary: Some(ExecutionStreamTerminalSummary {
standardized_usage: Some(standardized_usage),
finish_reason: None,
response_id: Some("resp_cancel_summary_1".to_string()),
model: Some("gpt-5.4".to_string()),
observed_finish: true,
unknown_event_count: 0,
parser_error: None,
}),
telemetry: None,
};
let event =
build_stream_terminal_usage_event(&plan, payload.report_context.as_ref(), &payload)
.expect("usage event should build");
assert_eq!(event.event_type, UsageEventType::Cancelled);
assert_eq!(event.data.input_tokens, Some(13));
assert_eq!(event.data.output_tokens, Some(21));
assert_eq!(event.data.total_tokens, Some(34));
assert_eq!(event.data.cache_creation_input_tokens, Some(2));
assert_eq!(event.data.cache_read_input_tokens, Some(3));
}
#[test]
@@ -3970,6 +3783,69 @@ mod tests {
assert!(event.data.client_response_body.is_none());
}
#[test]
fn stream_terminal_usage_marks_missing_observed_finish_as_failed() {
let plan = ExecutionPlan {
request_id: "req-stream-missing-finish-1".to_string(),
candidate_id: Some("cand-stream-missing-finish-1".to_string()),
provider_name: Some("OpenAI".to_string()),
provider_id: "provider-1".to_string(),
endpoint_id: "endpoint-1".to_string(),
key_id: "key-1".to_string(),
method: "POST".to_string(),
url: "https://example.com/v1/responses".to_string(),
headers: BTreeMap::new(),
content_type: None,
content_encoding: None,
body: RequestBody {
json_body: None,
body_bytes_b64: None,
body_ref: None,
},
stream: true,
client_api_format: "openai:responses".to_string(),
provider_api_format: "openai:responses".to_string(),
model_name: Some("gpt-5.5".to_string()),
proxy: None,
transport_profile: None,
timeouts: None,
};
let payload = GatewayStreamReportRequest {
trace_id: "trace-stream-missing-finish-1".to_string(),
report_kind: "openai_responses_stream_success".to_string(),
report_context: Some(json!({
"client_api_format": "openai:responses",
"provider_api_format": "openai:responses"
})),
status_code: 200,
headers: BTreeMap::new(),
provider_body_base64: None,
provider_body_state: Some(UsageBodyCaptureState::None),
client_body_base64: None,
client_body_state: Some(UsageBodyCaptureState::None),
terminal_summary: Some(ExecutionStreamTerminalSummary {
response_id: Some("resp_missing_finish".to_string()),
model: Some("gpt-5.5".to_string()),
observed_finish: false,
..ExecutionStreamTerminalSummary::default()
}),
telemetry: None,
};
let event =
build_stream_terminal_usage_event(&plan, payload.report_context.as_ref(), &payload)
.expect("usage event should build");
assert_eq!(event.event_type, UsageEventType::Failed);
assert_eq!(event.data.status_code, Some(200));
assert_eq!(
event.data.error_category.as_deref(),
Some("non_success_status")
);
assert_eq!(event.data.input_tokens, None);
assert_eq!(event.data.output_tokens, None);
}
#[test]
fn completed_image_usage_estimates_request_tokens_when_provider_usage_is_missing() {
let plan = ExecutionPlan {
@@ -5079,6 +4955,7 @@ mod tests {
..UsageRoutingSeed::default()
},
status_code: 200,
terminal_error_message: None,
response_time_ms: Some(123),
first_byte_time_ms: Some(45),
request_headers: Some(json!({