fix(pool): isolate dynamic model quota buckets and 429 scheduling

This commit is contained in:
zhefox
2026-09-02 15:23:23 +08:00
parent 2cd20da1ec
commit dbbe7b22ab
37 changed files with 1491 additions and 78 deletions
+1
View File
@@ -78,6 +78,7 @@ pub use crate::formats::openai::{
OpenAiProviderRequestFinalization,
},
responses::{
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
openai_responses_synthetic_reasoning_item_id,
strip_incompatible_openai_responses_reasoning_items,
strip_incompatible_openai_responses_reasoning_items_with_policy,
@@ -9,7 +9,7 @@ use serde_json::{json, Value};
use crate::formats::{
context::FormatContext,
openai::responses::{
openai_responses_synthetic_reasoning_item_id,
openai_responses_message_item_id, openai_responses_synthetic_reasoning_item_id,
response::ensure_modern_openai_responses_response_fields,
},
registry,
@@ -218,7 +218,7 @@ pub fn build_openai_responses_response_with_content(
if !content.is_empty() {
output.push(json!({
"type": "message",
"id": format!("{response_id}_msg"),
"id": openai_responses_message_item_id(response_id, 0),
"role": "assistant",
"status": "completed",
"content": content
@@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet};
use serde_json::{json, Map, Value};
use crate::formats::openai::responses::{
openai_responses_synthetic_reasoning_item_id,
openai_responses_message_item_id, openai_responses_synthetic_reasoning_item_id,
response::{
ensure_modern_openai_responses_response_fields, openai_responses_current_timestamp,
},
@@ -2240,7 +2240,7 @@ impl OpenAIResponsesClientEmitter {
fn message_item_id(&self) -> String {
self.message_item_id
.clone()
.unwrap_or_else(|| format!("{}_msg", self.response_id()))
.unwrap_or_else(|| openai_responses_message_item_id(self.response_id(), 0))
}
fn reasoning_item_id(&self) -> String {
@@ -2259,7 +2259,7 @@ impl OpenAIResponsesClientEmitter {
fn ensure_message_item_id(&mut self) -> String {
if self.message_item_id.is_none() {
self.message_item_id = Some(format!("{}_msg", self.response_id()));
self.message_item_id = Some(openai_responses_message_item_id(self.response_id(), 0));
}
self.message_item_id()
}
@@ -4167,7 +4167,7 @@ mod tests {
assert!(sse.contains("event: response.output_item.done\n"));
assert!(sse.contains("event: response.completed\n"));
assert!(sse.contains("\"response_id\":\"resp_stream_123\""));
assert!(sse.contains("\"item_id\":\"resp_stream_123_msg\""));
assert!(sse.contains("\"item_id\":\"msg_aether_"));
assert!(sse.contains("\"text\":\"Hello\""));
assert!(sse.contains("\"output_text\":\"Hello\""));
assert!(sse.contains("\"created_at\":"));
@@ -4231,8 +4231,8 @@ mod tests {
);
let sse = String::from_utf8(bytes).expect("sse should be utf8");
assert!(sse.contains("\"item_id\":\"msg_first_msg\""));
assert!(!sse.contains("\"item_id\":\"msg_second_msg\""));
assert!(sse.contains("\"item_id\":\"msg_aether_"));
assert!(!sse.contains("msg_second_msg"));
}
#[test]
@@ -147,6 +147,13 @@ fn finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_
finalization.provider_api_format,
reasoning_replay_policy,
);
if finalization
.provider_api_format
.trim()
.eq_ignore_ascii_case("openai:responses")
{
super::responses::normalize_openai_responses_message_item_ids(body);
}
crate::enforce_request_body_stream_field(
body,
finalization.provider_api_format,
@@ -408,6 +415,39 @@ mod tests {
assert_eq!(input[1]["type"], "message");
}
#[test]
fn finalization_repairs_legacy_responses_message_ids_for_same_format_upstream() {
let mut body = json!({
"model": "gpt-5.4",
"input": [{
"type": "message",
"id": "1c938e58-32a8-4d28-9c34-538d78076895_msg",
"role": "assistant",
"content": [{"type": "input_text", "text": "previous answer"}]
}]
});
finalize_openai_provider_request(
&mut body,
OpenAiProviderRequestFinalization {
source_api_format: "openai:responses",
provider_api_format: "openai:responses",
provider_type: "codex",
provider_model: "gpt-5.4",
source_model: "gpt-5.4",
body_rules: None,
upstream_is_stream: false,
require_body_stream_field: false,
},
)
.expect("legacy message IDs should be repaired before provider validation");
let repaired_id = body["input"][0]["id"]
.as_str()
.expect("message ID should be a string");
assert!(repaired_id.starts_with("msg_"));
}
#[test]
fn non_responses_sources_receive_codex_responses_reasoning_defaults() {
for source_api_format in ["openai:chat", "claude:messages", "gemini:generate_content"] {
@@ -9,6 +9,7 @@ pub mod stream;
const TOOL_ERROR_PREFIX: &str = "[tool error]";
const AETHER_REASONING_ITEM_ID_PREFIX: &str = "rs_aether_";
const AETHER_MESSAGE_ITEM_ID_PREFIX: &str = "msg_aether_";
/// Controls which provider-owned reasoning items may be replayed on a Responses request.
///
@@ -38,6 +39,67 @@ pub fn openai_responses_synthetic_reasoning_item_id(
)
}
/// Builds a stable, wire-compatible ID for a message item synthesized by Aether.
///
/// Responses clients replay assistant message items verbatim on the next turn and
/// OpenAI requires those IDs to begin with `msg`. Upstream response IDs are not
/// guaranteed to have that prefix (Chat completion IDs and UUIDs are common), so
/// appending a suffix to the response ID is not sufficient. A deterministic UUID
/// keeps the ID stable across sync/stream projections while avoiding assumptions
/// about the upstream ID's shape or length.
pub fn openai_responses_message_item_id(response_id: &str, output_index: usize) -> String {
let seed = format!("{response_id}:{output_index}");
format!(
"{AETHER_MESSAGE_ITEM_ID_PREFIX}{}",
uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, seed.as_bytes()).simple()
)
}
/// Repairs legacy/non-OpenAI message IDs in a Responses request in place.
///
/// Aether versions before the `msg_` contract emitted IDs such as
/// `<response-id>_msg`. Clients legitimately replay those assistant items on
/// the next turn, so merely fixing newly generated responses leaves existing
/// conversations broken. Preserve already-valid provider IDs and deterministically
/// remap only message items that do not begin with `msg`.
pub fn normalize_openai_responses_message_item_ids(body: &mut Value) -> usize {
let Some(items) = body.get_mut("input").and_then(Value::as_array_mut) else {
return 0;
};
let mut repaired = 0usize;
for (index, item) in items.iter_mut().enumerate() {
let Some(object) = item.as_object_mut() else {
continue;
};
if object.get("type").and_then(Value::as_str) != Some("message") {
continue;
}
let Some(raw_id) = object.get("id") else {
// IDs are optional for newly-authored input messages. Only repair
// an ID that a previous response actually supplied.
continue;
};
let valid = object
.get("id")
.and_then(Value::as_str)
.is_some_and(|id| id.starts_with("msg"));
if valid {
continue;
}
let source_id = raw_id
.as_str()
.filter(|id| !id.trim().is_empty())
.unwrap_or("missing")
.to_string();
object.insert(
"id".to_string(),
Value::String(openai_responses_message_item_id(source_id.as_str(), index)),
);
repaired += 1;
}
repaired
}
/// Removes reasoning history items that cannot be replayed against an OpenAI Responses backend.
///
/// Reasoning IDs are opaque provider references and must never be repaired by changing their
@@ -190,7 +252,9 @@ mod tests {
use serde_json::json;
use super::{
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
openai_responses_request_operation,
openai_responses_synthetic_reasoning_item_id,
strip_incompatible_openai_responses_reasoning_items,
strip_incompatible_openai_responses_reasoning_items_with_policy,
OpenAiResponsesReasoningReplayPolicy, OPENAI_RESPONSES_OPERATION_COMPACT,
@@ -238,6 +302,42 @@ mod tests {
assert_ne!(first, other);
}
#[test]
fn synthetic_message_item_ids_are_stable_and_start_with_msg() {
let first = openai_responses_message_item_id(
"1c938e58-32a8-4d28-9c34-538d78076895",
0,
);
let second = openai_responses_message_item_id(
"1c938e58-32a8-4d28-9c34-538d78076895",
0,
);
let other = openai_responses_message_item_id("chatcmpl-123", 1);
assert!(first.starts_with("msg_"));
assert_eq!(first, second);
assert_ne!(first, other);
}
#[test]
fn normalizes_legacy_message_ids_but_preserves_valid_ids() {
let mut body = json!({
"input": [
{"type": "message", "id": "1c938e58-32a8-4d28-9c34-538d78076895_msg", "role": "assistant"},
{"type": "message", "id": "msg_provider_123", "role": "assistant"},
{"type": "function_call", "id": "legacy_call"},
{"type": "message", "role": "user"}
]
});
assert_eq!(normalize_openai_responses_message_item_ids(&mut body), 1);
let input = body["input"].as_array().expect("input array");
assert!(input[0]["id"].as_str().is_some_and(|id| id.starts_with("msg_")));
assert_eq!(input[1]["id"], "msg_provider_123");
assert_eq!(input[2].get("id"), Some(&json!("legacy_call")));
assert!(input[3].get("id").is_none());
}
#[test]
fn strips_foreign_and_non_replayable_synthetic_reasoning_items() {
let portable_synthetic = openai_responses_synthetic_reasoning_item_id("resp_123", 1);
@@ -7,7 +7,9 @@ use aether_ai_formats::formats::conversion::response::{
convert_openai_chat_response_to_openai_responses,
convert_openai_responses_response_to_openai_chat,
};
use aether_ai_formats::formats::openai::responses::openai_responses_synthetic_reasoning_item_id;
use aether_ai_formats::formats::openai::responses::{
openai_responses_message_item_id, openai_responses_synthetic_reasoning_item_id,
};
use aether_ai_formats::formats::openai::responses::response::ensure_modern_openai_responses_response_fields;
use aether_ai_formats::formats::registry::{convert_response, FormatContext, FormatError};
use aether_ai_formats::{
@@ -2696,6 +2698,7 @@ fn aggregate_openai_responses_stream_sync_response_from_validated_terminal(
if let Some(state) = message_states.remove(&output_index) {
output.push(materialize_openai_responses_message_item(
&response_id,
output_index,
state,
));
}
@@ -3152,13 +3155,31 @@ fn resolve_openai_responses_tool_output_index(
fn materialize_openai_responses_message_item(
response_id: &str,
output_index: usize,
state: OpenAIResponsesSyncMessageState,
) -> Value {
let mut item = state.item;
item.entry("type".to_string())
.or_insert_with(|| Value::String("message".to_string()));
item.entry("id".to_string())
.or_insert_with(|| Value::String(format!("{response_id}_msg")));
let message_id_is_valid = item
.get("id")
.and_then(Value::as_str)
.is_some_and(|id| id.starts_with("msg"));
if !message_id_is_valid {
let source_id = item
.get("id")
.and_then(Value::as_str)
.filter(|id| !id.trim().is_empty())
.unwrap_or(response_id)
.to_string();
item.insert(
"id".to_string(),
Value::String(openai_responses_message_item_id(
source_id.as_str(),
output_index,
)),
);
}
item.entry("role".to_string())
.or_insert_with(|| Value::String("assistant".to_string()));
item.entry("status".to_string())
+3 -1
View File
@@ -56,7 +56,9 @@ pub use formats::openai::responses::request::{
validate_openai_responses_request_contract, OpenAiResponsesRequestContractViolation,
};
pub use formats::openai::responses::{
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
openai_responses_request_operation,
openai_responses_synthetic_reasoning_item_id,
strip_incompatible_openai_responses_reasoning_items,
strip_incompatible_openai_responses_reasoning_items_with_policy,
OpenAiResponsesReasoningReplayPolicy, OPENAI_RESPONSES_OPERATION_COMPACT,
@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use crate::formats::openai::shared::map_thinking_budget_to_openai_reasoning_effort;
use crate::formats::openai::responses::openai_responses_message_item_id;
use crate::formats::shared::model_directives::ReasoningEffort;
use crate::formats::shared::response::remove_empty_pages_from_tool_input_value;
@@ -3948,11 +3949,7 @@ pub(crate) fn flush_openai_responses_message_item(
if message_content.is_empty() {
return;
}
let id = if *message_index == 0 {
format!("{response_id}_msg")
} else {
format!("{response_id}_msg_{message_index}")
};
let id = openai_responses_message_item_id(response_id, *message_index);
output.push(json!({
"type": "message",
"id": id,