Merge remote-tracking branch 'origin/pr/593'

This commit is contained in:
elky
2026-06-02 21:40:59 +08:00
56 changed files with 1605 additions and 42 deletions
@@ -8,6 +8,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
"jina:embedding",
"gemini:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
];
fn unix_secs_to_rfc3339(unix_secs: u64) -> Option<String> {
+12
View File
@@ -835,6 +835,18 @@ const ADMIN_API_FORMAT_DEFINITIONS: &[AdminApiFormatDefinition] = &[
default_path: "/v1/embeddings",
aliases: &["doubao_embedding"],
},
AdminApiFormatDefinition {
value: "aliyun:multimodal_embedding",
label: "Aliyun Multimodal Embedding",
default_path: "/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding",
aliases: &[
"aliyun_embedding",
"aliyun_multimodal_embedding",
"dashscope_embedding",
"dashscope_multimodal_embedding",
"dashscope:multimodal_embedding",
],
},
];
pub fn build_admin_system_check_update_payload(current_version: String) -> serde_json::Value {
@@ -0,0 +1,2 @@
pub mod request;
pub mod response;
@@ -0,0 +1,256 @@
use serde_json::{Map, Value};
use crate::formats::context::FormatContext;
use crate::formats::openai::embedding::request::mapped_embedding_model;
use crate::protocol::canonical::{
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalRequest,
};
pub fn to(request: &CanonicalRequest, ctx: &FormatContext) -> Option<Value> {
let embedding = request.embedding.as_ref()?;
let contents = embedding_input_to_contents(&embedding.input)?;
if contents.is_empty() {
return None;
}
let mut output = Map::new();
output.insert(
"model".to_string(),
Value::String(mapped_embedding_model(
request,
ctx.mapped_model_or(request.model.as_str()),
)),
);
output.insert(
"input".to_string(),
Value::Object(Map::from_iter([(
"contents".to_string(),
Value::Array(contents),
)])),
);
let mut parameters = embedding.parameters.clone().unwrap_or_default();
if let Some(dimensions) = embedding.dimensions {
parameters
.entry("dimension".to_string())
.or_insert_with(|| Value::from(dimensions));
}
if !parameters.is_empty() {
output.insert("parameters".to_string(), Value::Object(parameters));
}
Some(Value::Object(output))
}
fn embedding_input_to_contents(input: &CanonicalEmbeddingInput) -> Option<Vec<Value>> {
match input {
CanonicalEmbeddingInput::String(text) => {
non_empty_text_content(text).map(|content| vec![content])
}
CanonicalEmbeddingInput::StringArray(items) => items
.iter()
.map(|text| non_empty_text_content(text))
.collect(),
CanonicalEmbeddingInput::Multimodal(items) => {
items.iter().map(multimodal_content_to_value).collect()
}
CanonicalEmbeddingInput::TokenArray(_) | CanonicalEmbeddingInput::TokenArrayArray(_) => {
None
}
}
}
fn non_empty_text_content(text: &str) -> Option<Value> {
let text = text.trim();
if text.is_empty() {
None
} else {
Some(Value::Object(Map::from_iter([(
"text".to_string(),
Value::String(text.to_string()),
)])))
}
}
fn multimodal_content_to_value(content: &CanonicalEmbeddingContent) -> Option<Value> {
if content.is_empty() {
return None;
}
let mut object = Map::new();
if let Some(text) = content
.text
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
object.insert("text".to_string(), Value::String(text.to_string()));
}
if let Some(image) = content
.image
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
object.insert("image".to_string(), Value::String(image.to_string()));
}
if let Some(video) = content
.video
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
object.insert("video".to_string(), Value::String(video.to_string()));
}
if let Some(multi_images) = content
.multi_images
.as_ref()
.filter(|values| !values.is_empty() && values.iter().all(|value| !value.trim().is_empty()))
{
object.insert(
"multi_images".to_string(),
Value::Array(
multi_images
.iter()
.map(|value| Value::String(value.trim().to_string()))
.collect(),
),
);
}
if object.is_empty() {
None
} else {
Some(Value::Object(object))
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use serde_json::{json, Map, Value};
use super::to;
use crate::formats::context::FormatContext;
use crate::protocol::canonical::{
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalEmbeddingRequest,
CanonicalRequest,
};
fn canonical_embedding(input: CanonicalEmbeddingInput) -> CanonicalRequest {
CanonicalRequest {
model: "text-embedding-3-small".to_string(),
embedding: Some(CanonicalEmbeddingRequest {
input,
encoding_format: None,
dimensions: None,
task: None,
user: None,
parameters: None,
extensions: BTreeMap::new(),
}),
..CanonicalRequest::default()
}
}
#[test]
fn text_input_uses_dashscope_contents() {
let request = canonical_embedding(CanonicalEmbeddingInput::StringArray(vec![
"alpha".to_string(),
"beta".to_string(),
]));
let body = to(
&request,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.expect("aliyun request");
assert_eq!(body["model"], "qwen3-vl-embedding");
assert_eq!(
body["input"]["contents"],
json!([{ "text": "alpha" }, { "text": "beta" }])
);
}
#[test]
fn multimodal_input_and_parameters_use_dashscope_contract() {
let mut request = canonical_embedding(CanonicalEmbeddingInput::Multimodal(vec![
CanonicalEmbeddingContent {
text: Some("white running shoes".to_string()),
image: None,
video: None,
multi_images: None,
},
CanonicalEmbeddingContent {
text: None,
image: Some("https://example.com/shoe.png".to_string()),
video: None,
multi_images: None,
},
CanonicalEmbeddingContent {
text: None,
image: None,
video: None,
multi_images: Some(vec![
"https://example.com/a.png".to_string(),
"https://example.com/b.png".to_string(),
]),
},
]));
let embedding = request.embedding.as_mut().expect("embedding request");
embedding.dimensions = Some(1024);
embedding.parameters = Some(Map::from_iter([
("enable_fusion".to_string(), Value::Bool(true)),
("res_level".to_string(), Value::from(2_u64)),
("max_video_frames".to_string(), Value::from(64_u64)),
]));
let body = to(
&request,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.expect("aliyun request");
assert_eq!(
body["input"]["contents"],
json!([
{ "text": "white running shoes" },
{ "image": "https://example.com/shoe.png" },
{ "multi_images": ["https://example.com/a.png", "https://example.com/b.png"] }
])
);
assert_eq!(body["parameters"]["dimension"], 1024);
assert_eq!(body["parameters"]["enable_fusion"], true);
assert_eq!(body["parameters"]["res_level"], 2);
assert_eq!(body["parameters"]["max_video_frames"], 64);
}
#[test]
fn parameter_dimension_wins_over_openai_dimensions() {
let mut request = canonical_embedding(CanonicalEmbeddingInput::String("alpha".to_string()));
let embedding = request.embedding.as_mut().expect("embedding request");
embedding.dimensions = Some(1024);
embedding.parameters = Some(Map::from_iter([(
"dimension".to_string(),
Value::from(512_u64),
)]));
let body = to(
&request,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.expect("aliyun request");
assert_eq!(body["parameters"]["dimension"], 512);
}
#[test]
fn token_arrays_are_not_convertible() {
let request = canonical_embedding(CanonicalEmbeddingInput::TokenArray(vec![1, 2, 3]));
assert!(to(
&request,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.is_none());
}
}
@@ -0,0 +1,134 @@
use std::collections::BTreeMap;
use serde_json::{Map, Value};
use crate::formats::openai::embedding::request::namespace_extensions;
use crate::protocol::canonical::{CanonicalEmbedding, CanonicalEmbeddingResponse, CanonicalUsage};
pub fn from(body_json: &Value) -> Option<CanonicalEmbeddingResponse> {
let body = body_json.as_object()?;
if body.contains_key("error") || body.contains_key("code") && body.contains_key("message") {
return None;
}
let data = body
.get("output")?
.as_object()?
.get("embeddings")?
.as_array()?;
let mut embeddings = Vec::new();
for (fallback_index, item) in data.iter().enumerate() {
let item_object = item.as_object()?;
let values = item_object.get("embedding")?.as_array()?;
let embedding = values
.iter()
.map(Value::as_f64)
.collect::<Option<Vec<_>>>()?;
let mut extensions =
namespace_extensions("aliyun", item_object, &["index", "embedding", "type"]);
if let Some(value) = item_object.get("type").cloned() {
extensions.insert(
"openai".to_string(),
Value::Object(Map::from_iter([("type".to_string(), value)])),
);
}
embeddings.push(CanonicalEmbedding {
index: item_object
.get("index")
.and_then(Value::as_u64)
.and_then(|value| usize::try_from(value).ok())
.unwrap_or(fallback_index),
embedding,
extensions,
});
}
let request_id = body.get("request_id").and_then(Value::as_str);
let mut extensions =
namespace_extensions("aliyun", body, &["output", "usage", "request_id", "model"]);
if let Some(request_id) = request_id {
extensions.insert(
"openai".to_string(),
Value::Object(Map::from_iter([(
"request_id".to_string(),
Value::String(request_id.to_string()),
)])),
);
}
Some(CanonicalEmbeddingResponse {
id: request_id.unwrap_or("aliyun-request-unknown").to_string(),
model: body
.get("model")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_string(),
embeddings,
usage: aliyun_usage_to_canonical(body.get("usage")),
extensions,
})
}
fn aliyun_usage_to_canonical(value: Option<&Value>) -> Option<CanonicalUsage> {
let usage = value?.as_object()?;
let input_tokens = usage
.get("input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let output_tokens = usage
.get("output_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
Some(CanonicalUsage {
input_tokens,
output_tokens,
total_tokens: usage
.get("total_tokens")
.and_then(Value::as_u64)
.unwrap_or(input_tokens.saturating_add(output_tokens)),
extensions: BTreeMap::from([("aliyun".to_string(), Value::Object(usage.clone()))]),
..CanonicalUsage::default()
})
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::from;
use crate::formats::openai::embedding::response::to as to_openai;
#[test]
fn parses_dashscope_embeddings_to_openai_compatible_shape() {
let body = json!({
"output": {
"embeddings": [
{
"index": 0,
"embedding": [0.1, 0.2, 0.3],
"type": "fused"
}
]
},
"usage": {
"input_tokens": 432,
"input_tokens_details": {
"image_tokens": 402,
"text_tokens": 30
},
"output_tokens": 1,
"total_tokens": 433
},
"request_id": "aliyun-request-1"
});
let canonical = from(&body).expect("aliyun response");
let emitted = to_openai(&canonical).expect("openai response");
assert_eq!(emitted["request_id"], "aliyun-request-1");
assert_eq!(emitted["data"][0]["embedding"], json!([0.1, 0.2, 0.3]));
assert_eq!(emitted["data"][0]["type"], "fused");
assert_eq!(emitted["usage"]["prompt_tokens"], 432);
assert_eq!(emitted["usage"]["completion_tokens"], 1);
assert_eq!(emitted["usage"]["total_tokens"], 433);
}
}
@@ -0,0 +1 @@
pub mod embedding;
@@ -113,6 +113,7 @@ mod tests {
dimensions: None,
task: None,
user: None,
parameters: None,
extensions: BTreeMap::new(),
}),
..CanonicalRequest::default()
+41 -2
View File
@@ -9,6 +9,7 @@ pub enum FormatFamily {
Gemini,
Jina,
Doubao,
Aliyun,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -30,6 +31,7 @@ pub enum FormatId {
JinaEmbedding,
JinaRerank,
DoubaoEmbedding,
AliyunMultimodalEmbedding,
}
impl FormatId {
@@ -52,6 +54,7 @@ impl FormatId {
Self::GeminiGenerateContent | Self::GeminiEmbedding => FormatFamily::Gemini,
Self::JinaEmbedding | Self::JinaRerank => FormatFamily::Jina,
Self::DoubaoEmbedding => FormatFamily::Doubao,
Self::AliyunMultimodalEmbedding => FormatFamily::Aliyun,
}
}
@@ -75,6 +78,7 @@ impl FormatId {
Self::JinaEmbedding => "jina:embedding",
Self::JinaRerank => "jina:rerank",
Self::DoubaoEmbedding => "doubao:embedding",
Self::AliyunMultimodalEmbedding => "aliyun:multimodal_embedding",
}
}
}
@@ -103,13 +107,22 @@ impl FromStr for FormatId {
"jina:embedding" | "/jina/v1/embeddings" => Ok(Self::JinaEmbedding),
"jina:rerank" | "/jina/v1/rerank" => Ok(Self::JinaRerank),
"doubao:embedding" => Ok(Self::DoubaoEmbedding),
"aliyun:multimodal_embedding"
| "aliyun_embedding"
| "aliyun_multimodal_embedding"
| "dashscope:multimodal_embedding"
| "dashscope_embedding"
| "dashscope_multimodal_embedding" => Ok(Self::AliyunMultimodalEmbedding),
_ => Err(()),
}
}
}
pub fn normalize_api_format_alias(value: &str) -> String {
value.trim().to_ascii_lowercase()
let normalized = value.trim().to_ascii_lowercase();
FormatId::parse(&normalized)
.map(|format| format.as_str().to_string())
.unwrap_or(normalized)
}
pub fn api_format_alias_matches(left: &str, right: &str) -> bool {
@@ -117,7 +130,13 @@ pub fn api_format_alias_matches(left: &str, right: &str) -> bool {
}
pub fn api_format_storage_aliases(value: &str) -> Vec<String> {
vec![normalize_api_format_alias(value)]
match FormatId::parse(value).map(FormatId::canonical) {
Some(FormatId::AliyunMultimodalEmbedding) => vec![
"aliyun:multimodal_embedding".to_string(),
"dashscope:multimodal_embedding".to_string(),
],
_ => vec![normalize_api_format_alias(value)],
}
}
pub fn is_openai_responses_format(value: &str) -> bool {
@@ -185,6 +204,18 @@ mod tests {
FormatId::parse("doubao:embedding"),
Some(FormatId::DoubaoEmbedding)
);
assert_eq!(
FormatId::parse("aliyun:multimodal_embedding").map(|format| format.to_string()),
Some("aliyun:multimodal_embedding".to_string())
);
assert_eq!(
FormatId::parse("dashscope:multimodal_embedding").map(|format| format.to_string()),
Some("aliyun:multimodal_embedding".to_string())
);
assert_eq!(
FormatId::parse("dashscope_embedding").map(|format| format.to_string()),
Some("aliyun:multimodal_embedding".to_string())
);
assert_eq!(FormatId::OpenAiEmbedding.to_string(), "openai:embedding");
}
@@ -197,6 +228,7 @@ mod tests {
(FormatId::GeminiEmbedding, FormatFamily::Gemini),
(FormatId::JinaEmbedding, FormatFamily::Jina),
(FormatId::DoubaoEmbedding, FormatFamily::Doubao),
(FormatId::AliyunMultimodalEmbedding, FormatFamily::Aliyun),
] {
assert_eq!(format.family(), family);
assert_eq!(format.profile(), FormatProfile::Default);
@@ -315,6 +347,13 @@ mod tests {
api_format_storage_aliases("doubao:embedding"),
vec!["doubao:embedding".to_string()]
);
assert_eq!(
api_format_storage_aliases("dashscope:multimodal_embedding"),
vec![
"aliyun:multimodal_embedding".to_string(),
"dashscope:multimodal_embedding".to_string(),
]
);
}
#[test]
+27 -4
View File
@@ -42,6 +42,7 @@ const EMBEDDING_CANDIDATE_API_FORMATS: &[&str] = &[
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
];
const RERANK_CANDIDATE_API_FORMATS: &[&str] = &["openai:rerank", "jina:rerank"];
@@ -238,7 +239,11 @@ pub fn is_standard_api_format(api_format: &str) -> bool {
pub fn is_embedding_api_format(api_format: &str) -> bool {
matches!(
normalize_api_format_alias(api_format).as_str(),
"openai:embedding" | "gemini:embedding" | "jina:embedding" | "doubao:embedding"
"openai:embedding"
| "gemini:embedding"
| "jina:embedding"
| "doubao:embedding"
| "aliyun:multimodal_embedding"
)
}
@@ -267,9 +272,11 @@ pub fn api_data_format_id(api_format: &str) -> Option<&'static str> {
"gemini:generate_content" => Some("gemini"),
"openai:chat" => Some("openai_chat"),
"openai:responses" | "openai:responses:compact" => Some("openai_responses"),
"openai:embedding" | "gemini:embedding" | "jina:embedding" | "doubao:embedding" => {
Some("embedding")
}
"openai:embedding"
| "gemini:embedding"
| "jina:embedding"
| "doubao:embedding"
| "aliyun:multimodal_embedding" => Some("embedding"),
"openai:rerank" | "jina:rerank" => Some("rerank"),
_ => None,
}
@@ -442,6 +449,7 @@ mod tests {
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
]
);
assert_eq!(
@@ -451,6 +459,7 @@ mod tests {
"openai:embedding",
"gemini:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
]
);
assert!(!request_candidate_api_formats("openai:embedding", false).contains(&"openai:chat"));
@@ -479,6 +488,7 @@ mod tests {
"openai:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
]
);
assert_eq!(
@@ -488,6 +498,17 @@ mod tests {
"openai:embedding",
"gemini:embedding",
"jina:embedding",
"aliyun:multimodal_embedding",
]
);
assert_eq!(
request_candidate_api_formats("aliyun:multimodal_embedding", false),
vec![
"aliyun:multimodal_embedding",
"openai:embedding",
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
]
);
@@ -496,6 +517,7 @@ mod tests {
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
];
for client_api_format in embedding_formats {
for provider_api_format in embedding_formats {
@@ -520,6 +542,7 @@ mod tests {
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
];
let standard_formats = [
"openai:chat",
@@ -1,3 +1,4 @@
pub mod aliyun;
pub mod claude;
pub mod context;
pub mod conversion;
@@ -50,6 +50,10 @@ pub(crate) fn from_namespace(body_json: &Value, namespace: &str) -> Option<Canon
.get("user")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
parameters: request
.get("parameters")
.and_then(Value::as_object)
.cloned(),
extensions: namespace_extensions(
namespace,
request,
@@ -60,6 +64,7 @@ pub(crate) fn from_namespace(body_json: &Value, namespace: &str) -> Option<Canon
"dimensions",
"task",
"user",
"parameters",
],
),
};
@@ -81,6 +86,9 @@ pub(crate) fn to_openai_like(
if embedding.input.is_empty() {
return None;
}
if matches!(&embedding.input, CanonicalEmbeddingInput::Multimodal(_)) {
return None;
}
let mut output = Map::new();
output.insert(
"model".to_string(),
@@ -99,6 +107,9 @@ pub(crate) fn to_openai_like(
if let Some(value) = &embedding.user {
output.insert("user".to_string(), Value::String(value.clone()));
}
if let Some(value) = &embedding.parameters {
output.insert("parameters".to_string(), Value::Object(value.clone()));
}
if let Some(task) = embedding
.task
.as_ref()
@@ -1,6 +1,7 @@
use serde_json::Value;
use crate::formats::{
aliyun,
claude::messages as claude_messages,
doubao,
gemini::{self, generate_content as gemini_generate_content},
@@ -29,7 +30,9 @@ pub fn parse_request(
FormatId::JinaEmbedding => jina::embedding::request::from(body, ctx),
FormatId::OpenAiRerank => openai::rerank::request::from(body, ctx),
FormatId::JinaRerank => jina::rerank::request::from(body, ctx),
FormatId::GeminiEmbedding | FormatId::DoubaoEmbedding => None,
FormatId::GeminiEmbedding
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => None,
}
.ok_or_else(|| FormatError::RequestParseFailed {
format: source.as_str().to_string(),
@@ -62,6 +65,7 @@ pub fn emit_request(
FormatId::JinaRerank => jina::rerank::request::to(&request, ctx),
FormatId::GeminiEmbedding => gemini::embedding::request::to(&request, ctx),
FormatId::DoubaoEmbedding => doubao::embedding::request::to(&request, ctx),
FormatId::AliyunMultimodalEmbedding => aliyun::embedding::request::to(&request, ctx),
}
.ok_or_else(|| FormatError::RequestEmitFailed {
format: target.as_str().to_string(),
@@ -96,7 +100,8 @@ pub fn parse_response(
| FormatId::OpenAiRerank
| FormatId::JinaRerank
| FormatId::GeminiEmbedding
| FormatId::DoubaoEmbedding => None,
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => None,
}
.ok_or_else(|| FormatError::ResponseParseFailed {
format: source.as_str().to_string(),
@@ -120,7 +125,8 @@ pub fn emit_response(
| FormatId::OpenAiRerank
| FormatId::JinaRerank
| FormatId::GeminiEmbedding
| FormatId::DoubaoEmbedding => None,
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => None,
}
.ok_or_else(|| FormatError::ResponseEmitFailed {
format: target.as_str().to_string(),
@@ -252,6 +258,119 @@ mod tests {
assert!(doubao.get("messages").is_none());
}
#[test]
fn converts_openai_embedding_to_aliyun_multimodal_payload_shape() {
let body = json!({
"model": "text-embedding-3-small",
"input": [
{"text": "white running shoes"},
{"image": "https://example.com/shoe.png"},
{"multi_images": ["https://example.com/a.png", "https://example.com/b.png"]}
],
"dimensions": 1024,
"parameters": {
"enable_fusion": true,
"res_level": 2,
"max_video_frames": 64
}
});
let converted = convert_request(
"openai:embedding",
"aliyun:multimodal_embedding",
&body,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.expect("aliyun multimodal embedding conversion should succeed");
assert_eq!(converted["model"], "qwen3-vl-embedding");
assert_eq!(converted["input"]["contents"], body["input"]);
assert_eq!(converted["parameters"]["dimension"], 1024);
assert_eq!(converted["parameters"]["enable_fusion"], true);
assert_eq!(converted["parameters"]["res_level"], 2);
assert_eq!(converted["parameters"]["max_video_frames"], 64);
assert!(converted.get("messages").is_none());
}
#[test]
fn aliyun_embedding_conversion_rejects_token_arrays() {
let body = json!({
"model": "text-embedding-3-small",
"input": [1, 2, 3]
});
assert!(convert_request(
"openai:embedding",
"aliyun:multimodal_embedding",
&body,
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
)
.is_err());
}
#[test]
fn multimodal_embedding_conversion_is_aliyun_only() {
let body = json!({
"model": "qwen3-vl-embedding",
"input": [
{"text": "white running shoes"},
{"image": "https://example.com/shoe.png"}
]
});
let ctx = FormatContext::default().with_mapped_model("qwen3-vl-embedding");
assert!(convert_request("openai:embedding", "openai:embedding", &body, &ctx).is_err());
assert!(convert_request("openai:embedding", "jina:embedding", &body, &ctx).is_err());
assert!(convert_request("openai:embedding", "gemini:embedding", &body, &ctx).is_err());
assert!(convert_request("openai:embedding", "doubao:embedding", &body, &ctx).is_err());
assert!(convert_request(
"openai:embedding",
"aliyun:multimodal_embedding",
&body,
&ctx
)
.is_ok());
}
#[test]
fn parses_aliyun_embedding_response_to_openai_shape() {
let body = json!({
"output": {
"embeddings": [
{
"index": 0,
"embedding": [0.1, 0.2, 0.3],
"type": "fused"
}
]
},
"usage": {
"input_tokens": 432,
"input_tokens_details": {
"image_tokens": 402,
"text_tokens": 30
},
"output_tokens": 1,
"total_tokens": 433
},
"request_id": "aliyun-request-1"
});
let canonical =
crate::protocol::canonical::from_embedding_to_canonical_response(&body, "aliyun")
.expect("aliyun embedding response should parse");
let emitted =
crate::protocol::canonical::canonical_to_embedding_response(&canonical, "openai")
.expect("openai embedding response should emit");
assert_eq!(emitted["request_id"], "aliyun-request-1");
assert_eq!(emitted["data"][0]["embedding"], json!([0.1, 0.2, 0.3]));
assert_eq!(emitted["data"][0]["type"], "fused");
assert_eq!(emitted["usage"]["prompt_tokens"], 432);
assert_eq!(emitted["usage"]["completion_tokens"], 1);
assert_eq!(emitted["usage"]["total_tokens"], 433);
}
#[test]
fn embedding_registry_keeps_gemini_and_doubao_emit_only() {
let body = json!({
@@ -260,7 +260,8 @@ impl ProviderStreamParser {
| FormatId::GeminiEmbedding
| FormatId::JinaEmbedding
| FormatId::JinaRerank
| FormatId::DoubaoEmbedding => return None,
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => return None,
})
}
@@ -350,7 +351,8 @@ impl ClientStreamEmitter {
| FormatId::GeminiEmbedding
| FormatId::JinaEmbedding
| FormatId::JinaRerank
| FormatId::DoubaoEmbedding => return None,
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => return None,
})
}
@@ -414,7 +416,8 @@ fn parse_provider_error(
| FormatId::GeminiEmbedding
| FormatId::JinaEmbedding
| FormatId::JinaRerank
| FormatId::DoubaoEmbedding => None,
| FormatId::DoubaoEmbedding
| FormatId::AliyunMultimodalEmbedding => None,
}
}
@@ -439,6 +439,7 @@ fn embedding_response_namespace_for_api_format(api_format: &str) -> Option<&'sta
"openai:embedding" => Some("openai"),
"jina:embedding" => Some("jina"),
"gemini:embedding" => Some("gemini"),
"aliyun:multimodal_embedding" => Some("aliyun"),
_ => None,
}
}
+5 -5
View File
@@ -43,9 +43,9 @@ pub use protocol::canonical::{
from_gemini_to_canonical_response, from_openai_chat_to_canonical_request,
from_openai_chat_to_canonical_response, from_openai_responses_to_canonical_request,
from_openai_responses_to_canonical_response, CanonicalContentBlock, CanonicalEmbedding,
CanonicalEmbeddingInput, CanonicalEmbeddingRequest, CanonicalEmbeddingResponse,
CanonicalGenerationConfig, CanonicalInstruction, CanonicalMessage, CanonicalRequest,
CanonicalResponse, CanonicalResponseFormat, CanonicalResponseOutput, CanonicalRole,
CanonicalStopReason, CanonicalStreamEvent, CanonicalStreamFrame, CanonicalThinkingConfig,
CanonicalToolChoice, CanonicalToolDefinition, CanonicalUsage,
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalEmbeddingRequest,
CanonicalEmbeddingResponse, CanonicalGenerationConfig, CanonicalInstruction, CanonicalMessage,
CanonicalRequest, CanonicalResponse, CanonicalResponseFormat, CanonicalResponseOutput,
CanonicalRole, CanonicalStopReason, CanonicalStreamEvent, CanonicalStreamFrame,
CanonicalThinkingConfig, CanonicalToolChoice, CanonicalToolDefinition, CanonicalUsage,
};
@@ -246,6 +246,19 @@ pub enum CanonicalEmbeddingInput {
StringArray(Vec<String>),
TokenArray(Vec<i64>),
TokenArrayArray(Vec<Vec<i64>>),
Multimodal(Vec<CanonicalEmbeddingContent>),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CanonicalEmbeddingContent {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub video: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub multi_images: Option<Vec<String>>,
}
impl CanonicalEmbeddingInput {
@@ -257,6 +270,9 @@ impl CanonicalEmbeddingInput {
}
Self::TokenArray(values) => values.is_empty(),
Self::TokenArrayArray(values) => values.is_empty() || values.iter().any(Vec::is_empty),
Self::Multimodal(values) => {
values.is_empty() || values.iter().any(CanonicalEmbeddingContent::is_empty)
}
}
}
@@ -264,11 +280,47 @@ impl CanonicalEmbeddingInput {
match self {
Self::String(value) => Some(vec![value.as_str()]),
Self::StringArray(values) => Some(values.iter().map(String::as_str).collect()),
Self::TokenArray(_) | Self::TokenArrayArray(_) => None,
Self::TokenArray(_) | Self::TokenArrayArray(_) | Self::Multimodal(_) => None,
}
}
}
impl CanonicalEmbeddingContent {
pub(crate) fn is_empty(&self) -> bool {
let text_empty = self
.text
.as_ref()
.is_some_and(|value| value.trim().is_empty());
let image_empty = self
.image
.as_ref()
.is_some_and(|value| value.trim().is_empty());
let video_empty = self
.video
.as_ref()
.is_some_and(|value| value.trim().is_empty());
let multi_images_empty = self.multi_images.as_ref().is_some_and(|values| {
values.is_empty() || values.iter().any(|value| value.trim().is_empty())
});
let has_any = self
.text
.as_ref()
.is_some_and(|value| !value.trim().is_empty())
|| self
.image
.as_ref()
.is_some_and(|value| !value.trim().is_empty())
|| self
.video
.as_ref()
.is_some_and(|value| !value.trim().is_empty())
|| self.multi_images.as_ref().is_some_and(|values| {
!values.is_empty() && values.iter().all(|value| !value.trim().is_empty())
});
!has_any || text_empty || image_empty || video_empty || multi_images_empty
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CanonicalEmbeddingRequest {
pub input: CanonicalEmbeddingInput,
@@ -280,6 +332,8 @@ pub struct CanonicalEmbeddingRequest {
pub task: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parameters: Option<Map<String, Value>>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub extensions: BTreeMap<String, Value>,
}
@@ -502,6 +556,7 @@ pub(crate) fn canonical_to_embedding_request(
"jina" => crate::formats::jina::embedding::request::to(canonical, &ctx),
"gemini" => crate::formats::gemini::embedding::request::to(canonical, &ctx),
"doubao" => crate::formats::doubao::embedding::request::to(canonical, &ctx),
"aliyun" => crate::formats::aliyun::embedding::request::to(canonical, &ctx),
_ => None,
}
}
@@ -699,6 +754,7 @@ pub fn from_embedding_to_canonical_response(
}
"jina" => crate::formats::openai::embedding::response::from_namespace(body_json, "jina"),
"gemini" => crate::formats::gemini::embedding::response::from(body_json),
"aliyun" => crate::formats::aliyun::embedding::response::from(body_json),
_ => None,
}
}
@@ -5244,8 +5300,8 @@ mod tests {
from_gemini_to_canonical_request, from_gemini_to_canonical_response,
from_openai_chat_to_canonical_request, from_openai_chat_to_canonical_response,
from_openai_responses_to_canonical_request, from_openai_responses_to_canonical_response,
CanonicalContentBlock, CanonicalEmbedding, CanonicalEmbeddingInput,
CanonicalEmbeddingRequest, CanonicalRole, CanonicalUsage,
CanonicalContentBlock, CanonicalEmbedding, CanonicalEmbeddingContent,
CanonicalEmbeddingInput, CanonicalEmbeddingRequest, CanonicalRole, CanonicalUsage,
};
use serde_json::{json, Value};
@@ -5301,6 +5357,44 @@ mod tests {
"nested token array",
CanonicalEmbeddingInput::TokenArrayArray(vec![vec![1, 2], vec![3, 4]]),
),
(
json!([
{"text": "white running shoes"},
{"image": "https://example.com/shoe.png"},
{"video": "https://example.com/demo.mp4"},
{"multi_images": ["https://example.com/a.png", "https://example.com/b.png"]}
]),
"multimodal array",
CanonicalEmbeddingInput::Multimodal(vec![
CanonicalEmbeddingContent {
text: Some("white running shoes".to_string()),
image: None,
video: None,
multi_images: None,
},
CanonicalEmbeddingContent {
text: None,
image: Some("https://example.com/shoe.png".to_string()),
video: None,
multi_images: None,
},
CanonicalEmbeddingContent {
text: None,
image: None,
video: Some("https://example.com/demo.mp4".to_string()),
multi_images: None,
},
CanonicalEmbeddingContent {
text: None,
image: None,
video: None,
multi_images: Some(vec![
"https://example.com/a.png".to_string(),
"https://example.com/b.png".to_string(),
]),
},
]),
),
];
for (input, label, expected_input) in cases {
@@ -5327,6 +5421,9 @@ mod tests {
json!({"model": "text-embedding-3-small", "input": []}),
json!({"model": "text-embedding-3-small", "input": [1, "two"]}),
json!({"model": "text-embedding-3-small", "input": [[1], []]}),
json!({"model": "text-embedding-3-small", "input": [{"image": " "}]}),
json!({"model": "text-embedding-3-small", "input": [{"multi_images": []}]}),
json!({"model": "text-embedding-3-small", "input": ["hello", {"image": "https://example.com/a.png"}]}),
json!({"model": "", "input": "hello"}),
json!({"input": "hello"}),
json!({"model": "text-embedding-3-small", "messages": []}),
@@ -5396,6 +5493,7 @@ mod tests {
dimensions: Some(2),
task: None,
user: None,
parameters: None,
extensions: Default::default(),
}),
..Default::default()
@@ -5441,6 +5539,7 @@ mod tests {
dimensions: Some(1536),
task: Some("retrieval.passage".to_string()),
user: Some("user-1".to_string()),
parameters: None,
extensions: Default::default(),
}),
..Default::default()
@@ -5493,6 +5592,7 @@ mod tests {
dimensions: None,
task: None,
user: None,
parameters: None,
extensions: Default::default(),
}),
..Default::default()
@@ -7,6 +7,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
"/v1/embeddings",
"/jina/v1/embeddings",
];
@@ -25,6 +25,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
"gemini:embedding",
"jina:embedding",
"doubao:embedding",
"aliyun:multimodal_embedding",
"/v1/embeddings",
"/jina/v1/embeddings",
];
@@ -71,6 +71,7 @@ SELECT
OR COALESCE(gm.config->'api_formats' @> '["jina:embedding"]'::jsonb, FALSE)
OR COALESCE(gm.config->'api_formats' @> '["gemini:embedding"]'::jsonb, FALSE)
OR COALESCE(gm.config->'api_formats' @> '["doubao:embedding"]'::jsonb, FALSE)
OR COALESCE(gm.config->'api_formats' @> '["aliyun:multimodal_embedding"]'::jsonb, FALSE)
OR LOWER(COALESCE(m.config->>'embedding', 'false')) = 'true'
OR LOWER(COALESCE(m.config->>'model_type', '')) = 'embedding'
OR LOWER(COALESCE(m.config->>'type', '')) = 'embedding'
@@ -80,6 +81,7 @@ SELECT
OR COALESCE(m.config::jsonb->'api_formats' @> '["jina:embedding"]'::jsonb, FALSE)
OR COALESCE(m.config::jsonb->'api_formats' @> '["gemini:embedding"]'::jsonb, FALSE)
OR COALESCE(m.config::jsonb->'api_formats' @> '["doubao:embedding"]'::jsonb, FALSE)
OR COALESCE(m.config::jsonb->'api_formats' @> '["aliyun:multimodal_embedding"]'::jsonb, FALSE)
) AS supports_embedding,
m.is_active
FROM models m
@@ -184,7 +184,11 @@ pub fn request_pair_transport_unsupported_reason(
)
}
}
"openai:embedding" | "jina:embedding" | "doubao:embedding" | "openai:rerank"
"openai:embedding"
| "jina:embedding"
| "doubao:embedding"
| "aliyun:multimodal_embedding"
| "openai:rerank"
| "jina:rerank" => local_standard_transport_unsupported_reason_with_network(
transport,
provider_api_format.as_str(),
@@ -221,6 +225,7 @@ fn request_direct_auth_for_provider_format(
| "openai:embedding"
| "jina:embedding"
| "doubao:embedding"
| "aliyun:multimodal_embedding"
| "openai:rerank"
| "jina:rerank" => resolve_local_openai_bearer_auth(transport),
"gemini:generate_content" | "gemini:embedding" => {
@@ -225,7 +225,7 @@ fn endpoint_kind_allows_embedding(endpoint_kind: Option<&str>) -> bool {
.map(|value| {
matches!(
value.to_ascii_lowercase().as_str(),
"embedding" | "embeddings"
"embedding" | "embeddings" | "multimodal_embedding" | "multimodal_embeddings"
)
})
.unwrap_or(true)
@@ -349,10 +349,13 @@ mod tests {
("jina", "jina:embedding"),
("doubao", "doubao:embedding"),
("volcengine", "doubao:embedding"),
("aliyun", "aliyun:multimodal_embedding"),
("dashscope", "aliyun:multimodal_embedding"),
("custom", "openai:embedding"),
("custom", "gemini:embedding"),
("custom", "jina:embedding"),
("custom", "doubao:embedding"),
("custom", "aliyun:multimodal_embedding"),
] {
let transport = sample_transport(provider_type, api_format, Some("embedding"));
assert_eq!(
@@ -84,6 +84,7 @@ pub enum ProviderLocalEmbeddingSupport {
Gemini,
Jina,
Doubao,
Aliyun,
}
impl ProviderLocalEmbeddingSupport {
@@ -99,11 +100,13 @@ impl ProviderLocalEmbeddingSupport {
| "jina:embedding"
| "jina:rerank"
| "doubao:embedding"
| "aliyun:multimodal_embedding"
),
Self::OpenAi => matches!(api_format.as_str(), "openai:embedding" | "openai:rerank"),
Self::Gemini => api_format == "gemini:embedding",
Self::Jina => matches!(api_format.as_str(), "jina:embedding" | "jina:rerank"),
Self::Doubao => api_format == "doubao:embedding",
Self::Aliyun => api_format == "aliyun:multimodal_embedding",
}
}
}
@@ -192,6 +195,10 @@ const DOUBAO_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
local_embedding_support: ProviderLocalEmbeddingSupport::Doubao,
..STANDARD_RUNTIME_POLICY
};
const ALIYUN_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
local_embedding_support: ProviderLocalEmbeddingSupport::Aliyun,
..STANDARD_RUNTIME_POLICY
};
const CLAUDE_CODE_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
fixed_provider: true,
@@ -470,6 +477,7 @@ pub fn provider_runtime_policy(provider_type: &str) -> ProviderRuntimePolicy {
"gemini" | "google" => GEMINI_RUNTIME_POLICY,
"jina" => JINA_RUNTIME_POLICY,
"doubao" | "volcengine" => DOUBAO_RUNTIME_POLICY,
"aliyun" | "dashscope" => ALIYUN_RUNTIME_POLICY,
_ => STANDARD_RUNTIME_POLICY,
}
}
@@ -875,6 +883,8 @@ mod tests {
("jina", "jina:embedding"),
("doubao", "doubao:embedding"),
("volcengine", "doubao:embedding"),
("aliyun", "aliyun:multimodal_embedding"),
("dashscope", "aliyun:multimodal_embedding"),
] {
assert!(
provider_type_supports_local_embedding_transport(provider_type, api_format),
@@ -888,6 +898,8 @@ mod tests {
("vertex_ai", "openai:embedding"),
("jina", "doubao:embedding"),
("doubao", "jina:embedding"),
("aliyun", "openai:embedding"),
("openai", "aliyun:multimodal_embedding"),
("claude_code", "openai:embedding"),
("openai", "openai:chat"),
] {
@@ -127,6 +127,10 @@ fn build_transport_request_url_inner(
"openai:embedding" | "jina:embedding" => {
build_provider_embedding_v1_url(&transport.endpoint.base_url, params.request_query)
}
"aliyun:multimodal_embedding" => build_aliyun_multimodal_embedding_url(
&transport.endpoint.base_url,
params.request_query,
),
"openai:rerank" | "jina:rerank" => {
build_provider_rerank_v1_url(&transport.endpoint.base_url, params.request_query)
}
@@ -425,6 +429,18 @@ fn build_provider_embedding_v1_url(upstream_base_url: &str, query: Option<&str>)
build_provider_api_root_url(upstream_base_url, "/embeddings", query)
}
fn build_aliyun_multimodal_embedding_url(
upstream_base_url: &str,
query: Option<&str>,
) -> Option<String> {
build_passthrough_path_url(
upstream_base_url,
"/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding",
query,
&[],
)
}
fn build_provider_rerank_v1_url(upstream_base_url: &str, query: Option<&str>) -> Option<String> {
build_provider_api_root_url(upstream_base_url, "/rerank", query)
}
@@ -1019,6 +1035,12 @@ mod tests {
"https://ark.volces.example/api/v3",
None,
);
let aliyun = sample_transport(
"aliyun",
"aliyun:multimodal_embedding",
"https://dashscope.aliyuncs.com",
None,
);
assert_eq!(
build_transport_request_url(
@@ -1078,6 +1100,20 @@ mod tests {
.as_deref(),
Some("https://ark.volces.example/api/v3/embeddings")
);
assert_eq!(
build_transport_request_url(
&aliyun,
TransportRequestUrlParams {
provider_api_format: "aliyun:multimodal_embedding",
mapped_model: Some("qwen3-vl-embedding"),
upstream_is_stream: false,
request_query: None,
kiro_api_region: None,
},
)
.as_deref(),
Some("https://dashscope.aliyuncs.com/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding")
);
}
#[test]
@@ -169,6 +169,14 @@ pub fn build_same_format_provider_request_body(
);
}
if embedding_multimodal_input_requires_aliyun_provider(
input.client_api_format,
input.provider_api_format,
input.body_json,
) {
return None;
}
let mut provider_request_body = if aether_ai_formats::api_format_alias_matches(
input.client_api_format,
input.provider_api_format,
@@ -245,6 +253,31 @@ pub fn build_same_format_provider_request_body(
Some(provider_request_body)
}
fn embedding_multimodal_input_requires_aliyun_provider(
client_api_format: &str,
provider_api_format: &str,
body_json: &Value,
) -> bool {
aether_ai_formats::is_embedding_api_format(client_api_format)
&& embedding_input_is_multimodal(body_json.get("input"))
&& aether_ai_formats::normalize_api_format_alias(provider_api_format)
!= "aliyun:multimodal_embedding"
}
fn embedding_input_is_multimodal(value: Option<&Value>) -> bool {
value
.and_then(Value::as_array)
.is_some_and(|items| !items.is_empty() && items.iter().all(embedding_content_is_multimodal))
}
fn embedding_content_is_multimodal(value: &Value) -> bool {
value.as_object().is_some_and(|object| {
["text", "image", "video", "multi_images"]
.iter()
.any(|key| object.contains_key(*key))
})
}
fn strip_gemini_function_response_ids(value: &mut Value) {
match value {
Value::Object(object) => {
@@ -849,6 +882,33 @@ mod tests {
assert_eq!(body.get("stream"), Some(&json!(true)));
}
#[test]
fn same_format_embedding_body_rejects_multimodal_for_openai_like_provider() {
let body = build_same_format_provider_request_body(SameFormatProviderRequestBodyInput {
body_json: &json!({
"model": "qwen3-vl-embedding",
"input": [
{"text": "white running shoes"},
{"image": "https://example.com/shoe.png"}
]
}),
mapped_model: "openai-qwen-fallback",
client_api_format: "openai:embedding",
provider_api_format: "openai:embedding",
source_model: Some("qwen3-vl-embedding"),
family: SameFormatProviderFamily::Standard,
body_rules: None,
request_headers: None,
upstream_is_stream: false,
force_body_stream_field: false,
kiro_auth_config: None,
is_claude_code: false,
enable_model_directives: false,
});
assert!(body.is_none());
}
#[test]
fn same_format_standard_body_overrides_client_stream_for_non_stream_upstream() {
let body = build_same_format_provider_request_body(SameFormatProviderRequestBodyInput {