mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-12 06:00:20 +08:00
Merge remote-tracking branch 'origin/pr/593'
This commit is contained in:
@@ -8,6 +8,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
|
||||
"jina:embedding",
|
||||
"gemini:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
];
|
||||
|
||||
fn unix_secs_to_rfc3339(unix_secs: u64) -> Option<String> {
|
||||
|
||||
@@ -835,6 +835,18 @@ const ADMIN_API_FORMAT_DEFINITIONS: &[AdminApiFormatDefinition] = &[
|
||||
default_path: "/v1/embeddings",
|
||||
aliases: &["doubao_embedding"],
|
||||
},
|
||||
AdminApiFormatDefinition {
|
||||
value: "aliyun:multimodal_embedding",
|
||||
label: "Aliyun Multimodal Embedding",
|
||||
default_path: "/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding",
|
||||
aliases: &[
|
||||
"aliyun_embedding",
|
||||
"aliyun_multimodal_embedding",
|
||||
"dashscope_embedding",
|
||||
"dashscope_multimodal_embedding",
|
||||
"dashscope:multimodal_embedding",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
pub fn build_admin_system_check_update_payload(current_version: String) -> serde_json::Value {
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod request;
|
||||
pub mod response;
|
||||
@@ -0,0 +1,256 @@
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::formats::context::FormatContext;
|
||||
use crate::formats::openai::embedding::request::mapped_embedding_model;
|
||||
use crate::protocol::canonical::{
|
||||
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalRequest,
|
||||
};
|
||||
|
||||
pub fn to(request: &CanonicalRequest, ctx: &FormatContext) -> Option<Value> {
|
||||
let embedding = request.embedding.as_ref()?;
|
||||
let contents = embedding_input_to_contents(&embedding.input)?;
|
||||
if contents.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut output = Map::new();
|
||||
output.insert(
|
||||
"model".to_string(),
|
||||
Value::String(mapped_embedding_model(
|
||||
request,
|
||||
ctx.mapped_model_or(request.model.as_str()),
|
||||
)),
|
||||
);
|
||||
output.insert(
|
||||
"input".to_string(),
|
||||
Value::Object(Map::from_iter([(
|
||||
"contents".to_string(),
|
||||
Value::Array(contents),
|
||||
)])),
|
||||
);
|
||||
|
||||
let mut parameters = embedding.parameters.clone().unwrap_or_default();
|
||||
if let Some(dimensions) = embedding.dimensions {
|
||||
parameters
|
||||
.entry("dimension".to_string())
|
||||
.or_insert_with(|| Value::from(dimensions));
|
||||
}
|
||||
if !parameters.is_empty() {
|
||||
output.insert("parameters".to_string(), Value::Object(parameters));
|
||||
}
|
||||
|
||||
Some(Value::Object(output))
|
||||
}
|
||||
|
||||
fn embedding_input_to_contents(input: &CanonicalEmbeddingInput) -> Option<Vec<Value>> {
|
||||
match input {
|
||||
CanonicalEmbeddingInput::String(text) => {
|
||||
non_empty_text_content(text).map(|content| vec![content])
|
||||
}
|
||||
CanonicalEmbeddingInput::StringArray(items) => items
|
||||
.iter()
|
||||
.map(|text| non_empty_text_content(text))
|
||||
.collect(),
|
||||
CanonicalEmbeddingInput::Multimodal(items) => {
|
||||
items.iter().map(multimodal_content_to_value).collect()
|
||||
}
|
||||
CanonicalEmbeddingInput::TokenArray(_) | CanonicalEmbeddingInput::TokenArrayArray(_) => {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty_text_content(text: &str) -> Option<Value> {
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Value::Object(Map::from_iter([(
|
||||
"text".to_string(),
|
||||
Value::String(text.to_string()),
|
||||
)])))
|
||||
}
|
||||
}
|
||||
|
||||
fn multimodal_content_to_value(content: &CanonicalEmbeddingContent) -> Option<Value> {
|
||||
if content.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut object = Map::new();
|
||||
if let Some(text) = content
|
||||
.text
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
object.insert("text".to_string(), Value::String(text.to_string()));
|
||||
}
|
||||
if let Some(image) = content
|
||||
.image
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
object.insert("image".to_string(), Value::String(image.to_string()));
|
||||
}
|
||||
if let Some(video) = content
|
||||
.video
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
object.insert("video".to_string(), Value::String(video.to_string()));
|
||||
}
|
||||
if let Some(multi_images) = content
|
||||
.multi_images
|
||||
.as_ref()
|
||||
.filter(|values| !values.is_empty() && values.iter().all(|value| !value.trim().is_empty()))
|
||||
{
|
||||
object.insert(
|
||||
"multi_images".to_string(),
|
||||
Value::Array(
|
||||
multi_images
|
||||
.iter()
|
||||
.map(|value| Value::String(value.trim().to_string()))
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
}
|
||||
if object.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Value::Object(object))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::to;
|
||||
use crate::formats::context::FormatContext;
|
||||
use crate::protocol::canonical::{
|
||||
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalEmbeddingRequest,
|
||||
CanonicalRequest,
|
||||
};
|
||||
|
||||
fn canonical_embedding(input: CanonicalEmbeddingInput) -> CanonicalRequest {
|
||||
CanonicalRequest {
|
||||
model: "text-embedding-3-small".to_string(),
|
||||
embedding: Some(CanonicalEmbeddingRequest {
|
||||
input,
|
||||
encoding_format: None,
|
||||
dimensions: None,
|
||||
task: None,
|
||||
user: None,
|
||||
parameters: None,
|
||||
extensions: BTreeMap::new(),
|
||||
}),
|
||||
..CanonicalRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_input_uses_dashscope_contents() {
|
||||
let request = canonical_embedding(CanonicalEmbeddingInput::StringArray(vec![
|
||||
"alpha".to_string(),
|
||||
"beta".to_string(),
|
||||
]));
|
||||
|
||||
let body = to(
|
||||
&request,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.expect("aliyun request");
|
||||
|
||||
assert_eq!(body["model"], "qwen3-vl-embedding");
|
||||
assert_eq!(
|
||||
body["input"]["contents"],
|
||||
json!([{ "text": "alpha" }, { "text": "beta" }])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_input_and_parameters_use_dashscope_contract() {
|
||||
let mut request = canonical_embedding(CanonicalEmbeddingInput::Multimodal(vec![
|
||||
CanonicalEmbeddingContent {
|
||||
text: Some("white running shoes".to_string()),
|
||||
image: None,
|
||||
video: None,
|
||||
multi_images: None,
|
||||
},
|
||||
CanonicalEmbeddingContent {
|
||||
text: None,
|
||||
image: Some("https://example.com/shoe.png".to_string()),
|
||||
video: None,
|
||||
multi_images: None,
|
||||
},
|
||||
CanonicalEmbeddingContent {
|
||||
text: None,
|
||||
image: None,
|
||||
video: None,
|
||||
multi_images: Some(vec![
|
||||
"https://example.com/a.png".to_string(),
|
||||
"https://example.com/b.png".to_string(),
|
||||
]),
|
||||
},
|
||||
]));
|
||||
let embedding = request.embedding.as_mut().expect("embedding request");
|
||||
embedding.dimensions = Some(1024);
|
||||
embedding.parameters = Some(Map::from_iter([
|
||||
("enable_fusion".to_string(), Value::Bool(true)),
|
||||
("res_level".to_string(), Value::from(2_u64)),
|
||||
("max_video_frames".to_string(), Value::from(64_u64)),
|
||||
]));
|
||||
|
||||
let body = to(
|
||||
&request,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.expect("aliyun request");
|
||||
|
||||
assert_eq!(
|
||||
body["input"]["contents"],
|
||||
json!([
|
||||
{ "text": "white running shoes" },
|
||||
{ "image": "https://example.com/shoe.png" },
|
||||
{ "multi_images": ["https://example.com/a.png", "https://example.com/b.png"] }
|
||||
])
|
||||
);
|
||||
assert_eq!(body["parameters"]["dimension"], 1024);
|
||||
assert_eq!(body["parameters"]["enable_fusion"], true);
|
||||
assert_eq!(body["parameters"]["res_level"], 2);
|
||||
assert_eq!(body["parameters"]["max_video_frames"], 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_dimension_wins_over_openai_dimensions() {
|
||||
let mut request = canonical_embedding(CanonicalEmbeddingInput::String("alpha".to_string()));
|
||||
let embedding = request.embedding.as_mut().expect("embedding request");
|
||||
embedding.dimensions = Some(1024);
|
||||
embedding.parameters = Some(Map::from_iter([(
|
||||
"dimension".to_string(),
|
||||
Value::from(512_u64),
|
||||
)]));
|
||||
|
||||
let body = to(
|
||||
&request,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.expect("aliyun request");
|
||||
|
||||
assert_eq!(body["parameters"]["dimension"], 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_arrays_are_not_convertible() {
|
||||
let request = canonical_embedding(CanonicalEmbeddingInput::TokenArray(vec![1, 2, 3]));
|
||||
assert!(to(
|
||||
&request,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::formats::openai::embedding::request::namespace_extensions;
|
||||
use crate::protocol::canonical::{CanonicalEmbedding, CanonicalEmbeddingResponse, CanonicalUsage};
|
||||
|
||||
pub fn from(body_json: &Value) -> Option<CanonicalEmbeddingResponse> {
|
||||
let body = body_json.as_object()?;
|
||||
if body.contains_key("error") || body.contains_key("code") && body.contains_key("message") {
|
||||
return None;
|
||||
}
|
||||
let data = body
|
||||
.get("output")?
|
||||
.as_object()?
|
||||
.get("embeddings")?
|
||||
.as_array()?;
|
||||
let mut embeddings = Vec::new();
|
||||
for (fallback_index, item) in data.iter().enumerate() {
|
||||
let item_object = item.as_object()?;
|
||||
let values = item_object.get("embedding")?.as_array()?;
|
||||
let embedding = values
|
||||
.iter()
|
||||
.map(Value::as_f64)
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
let mut extensions =
|
||||
namespace_extensions("aliyun", item_object, &["index", "embedding", "type"]);
|
||||
if let Some(value) = item_object.get("type").cloned() {
|
||||
extensions.insert(
|
||||
"openai".to_string(),
|
||||
Value::Object(Map::from_iter([("type".to_string(), value)])),
|
||||
);
|
||||
}
|
||||
embeddings.push(CanonicalEmbedding {
|
||||
index: item_object
|
||||
.get("index")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|value| usize::try_from(value).ok())
|
||||
.unwrap_or(fallback_index),
|
||||
embedding,
|
||||
extensions,
|
||||
});
|
||||
}
|
||||
|
||||
let request_id = body.get("request_id").and_then(Value::as_str);
|
||||
let mut extensions =
|
||||
namespace_extensions("aliyun", body, &["output", "usage", "request_id", "model"]);
|
||||
if let Some(request_id) = request_id {
|
||||
extensions.insert(
|
||||
"openai".to_string(),
|
||||
Value::Object(Map::from_iter([(
|
||||
"request_id".to_string(),
|
||||
Value::String(request_id.to_string()),
|
||||
)])),
|
||||
);
|
||||
}
|
||||
|
||||
Some(CanonicalEmbeddingResponse {
|
||||
id: request_id.unwrap_or("aliyun-request-unknown").to_string(),
|
||||
model: body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("unknown")
|
||||
.to_string(),
|
||||
embeddings,
|
||||
usage: aliyun_usage_to_canonical(body.get("usage")),
|
||||
extensions,
|
||||
})
|
||||
}
|
||||
|
||||
fn aliyun_usage_to_canonical(value: Option<&Value>) -> Option<CanonicalUsage> {
|
||||
let usage = value?.as_object()?;
|
||||
let input_tokens = usage
|
||||
.get("input_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let output_tokens = usage
|
||||
.get("output_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
Some(CanonicalUsage {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
total_tokens: usage
|
||||
.get("total_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(input_tokens.saturating_add(output_tokens)),
|
||||
extensions: BTreeMap::from([("aliyun".to_string(), Value::Object(usage.clone()))]),
|
||||
..CanonicalUsage::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::from;
|
||||
use crate::formats::openai::embedding::response::to as to_openai;
|
||||
|
||||
#[test]
|
||||
fn parses_dashscope_embeddings_to_openai_compatible_shape() {
|
||||
let body = json!({
|
||||
"output": {
|
||||
"embeddings": [
|
||||
{
|
||||
"index": 0,
|
||||
"embedding": [0.1, 0.2, 0.3],
|
||||
"type": "fused"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input_tokens": 432,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": 402,
|
||||
"text_tokens": 30
|
||||
},
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 433
|
||||
},
|
||||
"request_id": "aliyun-request-1"
|
||||
});
|
||||
|
||||
let canonical = from(&body).expect("aliyun response");
|
||||
let emitted = to_openai(&canonical).expect("openai response");
|
||||
|
||||
assert_eq!(emitted["request_id"], "aliyun-request-1");
|
||||
assert_eq!(emitted["data"][0]["embedding"], json!([0.1, 0.2, 0.3]));
|
||||
assert_eq!(emitted["data"][0]["type"], "fused");
|
||||
assert_eq!(emitted["usage"]["prompt_tokens"], 432);
|
||||
assert_eq!(emitted["usage"]["completion_tokens"], 1);
|
||||
assert_eq!(emitted["usage"]["total_tokens"], 433);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod embedding;
|
||||
@@ -113,6 +113,7 @@ mod tests {
|
||||
dimensions: None,
|
||||
task: None,
|
||||
user: None,
|
||||
parameters: None,
|
||||
extensions: BTreeMap::new(),
|
||||
}),
|
||||
..CanonicalRequest::default()
|
||||
|
||||
@@ -9,6 +9,7 @@ pub enum FormatFamily {
|
||||
Gemini,
|
||||
Jina,
|
||||
Doubao,
|
||||
Aliyun,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
@@ -30,6 +31,7 @@ pub enum FormatId {
|
||||
JinaEmbedding,
|
||||
JinaRerank,
|
||||
DoubaoEmbedding,
|
||||
AliyunMultimodalEmbedding,
|
||||
}
|
||||
|
||||
impl FormatId {
|
||||
@@ -52,6 +54,7 @@ impl FormatId {
|
||||
Self::GeminiGenerateContent | Self::GeminiEmbedding => FormatFamily::Gemini,
|
||||
Self::JinaEmbedding | Self::JinaRerank => FormatFamily::Jina,
|
||||
Self::DoubaoEmbedding => FormatFamily::Doubao,
|
||||
Self::AliyunMultimodalEmbedding => FormatFamily::Aliyun,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +78,7 @@ impl FormatId {
|
||||
Self::JinaEmbedding => "jina:embedding",
|
||||
Self::JinaRerank => "jina:rerank",
|
||||
Self::DoubaoEmbedding => "doubao:embedding",
|
||||
Self::AliyunMultimodalEmbedding => "aliyun:multimodal_embedding",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -103,13 +107,22 @@ impl FromStr for FormatId {
|
||||
"jina:embedding" | "/jina/v1/embeddings" => Ok(Self::JinaEmbedding),
|
||||
"jina:rerank" | "/jina/v1/rerank" => Ok(Self::JinaRerank),
|
||||
"doubao:embedding" => Ok(Self::DoubaoEmbedding),
|
||||
"aliyun:multimodal_embedding"
|
||||
| "aliyun_embedding"
|
||||
| "aliyun_multimodal_embedding"
|
||||
| "dashscope:multimodal_embedding"
|
||||
| "dashscope_embedding"
|
||||
| "dashscope_multimodal_embedding" => Ok(Self::AliyunMultimodalEmbedding),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_api_format_alias(value: &str) -> String {
|
||||
value.trim().to_ascii_lowercase()
|
||||
let normalized = value.trim().to_ascii_lowercase();
|
||||
FormatId::parse(&normalized)
|
||||
.map(|format| format.as_str().to_string())
|
||||
.unwrap_or(normalized)
|
||||
}
|
||||
|
||||
pub fn api_format_alias_matches(left: &str, right: &str) -> bool {
|
||||
@@ -117,7 +130,13 @@ pub fn api_format_alias_matches(left: &str, right: &str) -> bool {
|
||||
}
|
||||
|
||||
pub fn api_format_storage_aliases(value: &str) -> Vec<String> {
|
||||
vec![normalize_api_format_alias(value)]
|
||||
match FormatId::parse(value).map(FormatId::canonical) {
|
||||
Some(FormatId::AliyunMultimodalEmbedding) => vec![
|
||||
"aliyun:multimodal_embedding".to_string(),
|
||||
"dashscope:multimodal_embedding".to_string(),
|
||||
],
|
||||
_ => vec![normalize_api_format_alias(value)],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_openai_responses_format(value: &str) -> bool {
|
||||
@@ -185,6 +204,18 @@ mod tests {
|
||||
FormatId::parse("doubao:embedding"),
|
||||
Some(FormatId::DoubaoEmbedding)
|
||||
);
|
||||
assert_eq!(
|
||||
FormatId::parse("aliyun:multimodal_embedding").map(|format| format.to_string()),
|
||||
Some("aliyun:multimodal_embedding".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
FormatId::parse("dashscope:multimodal_embedding").map(|format| format.to_string()),
|
||||
Some("aliyun:multimodal_embedding".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
FormatId::parse("dashscope_embedding").map(|format| format.to_string()),
|
||||
Some("aliyun:multimodal_embedding".to_string())
|
||||
);
|
||||
assert_eq!(FormatId::OpenAiEmbedding.to_string(), "openai:embedding");
|
||||
}
|
||||
|
||||
@@ -197,6 +228,7 @@ mod tests {
|
||||
(FormatId::GeminiEmbedding, FormatFamily::Gemini),
|
||||
(FormatId::JinaEmbedding, FormatFamily::Jina),
|
||||
(FormatId::DoubaoEmbedding, FormatFamily::Doubao),
|
||||
(FormatId::AliyunMultimodalEmbedding, FormatFamily::Aliyun),
|
||||
] {
|
||||
assert_eq!(format.family(), family);
|
||||
assert_eq!(format.profile(), FormatProfile::Default);
|
||||
@@ -315,6 +347,13 @@ mod tests {
|
||||
api_format_storage_aliases("doubao:embedding"),
|
||||
vec!["doubao:embedding".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
api_format_storage_aliases("dashscope:multimodal_embedding"),
|
||||
vec![
|
||||
"aliyun:multimodal_embedding".to_string(),
|
||||
"dashscope:multimodal_embedding".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -42,6 +42,7 @@ const EMBEDDING_CANDIDATE_API_FORMATS: &[&str] = &[
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
];
|
||||
const RERANK_CANDIDATE_API_FORMATS: &[&str] = &["openai:rerank", "jina:rerank"];
|
||||
|
||||
@@ -238,7 +239,11 @@ pub fn is_standard_api_format(api_format: &str) -> bool {
|
||||
pub fn is_embedding_api_format(api_format: &str) -> bool {
|
||||
matches!(
|
||||
normalize_api_format_alias(api_format).as_str(),
|
||||
"openai:embedding" | "gemini:embedding" | "jina:embedding" | "doubao:embedding"
|
||||
"openai:embedding"
|
||||
| "gemini:embedding"
|
||||
| "jina:embedding"
|
||||
| "doubao:embedding"
|
||||
| "aliyun:multimodal_embedding"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -267,9 +272,11 @@ pub fn api_data_format_id(api_format: &str) -> Option<&'static str> {
|
||||
"gemini:generate_content" => Some("gemini"),
|
||||
"openai:chat" => Some("openai_chat"),
|
||||
"openai:responses" | "openai:responses:compact" => Some("openai_responses"),
|
||||
"openai:embedding" | "gemini:embedding" | "jina:embedding" | "doubao:embedding" => {
|
||||
Some("embedding")
|
||||
}
|
||||
"openai:embedding"
|
||||
| "gemini:embedding"
|
||||
| "jina:embedding"
|
||||
| "doubao:embedding"
|
||||
| "aliyun:multimodal_embedding" => Some("embedding"),
|
||||
"openai:rerank" | "jina:rerank" => Some("rerank"),
|
||||
_ => None,
|
||||
}
|
||||
@@ -442,6 +449,7 @@ mod tests {
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -451,6 +459,7 @@ mod tests {
|
||||
"openai:embedding",
|
||||
"gemini:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
]
|
||||
);
|
||||
assert!(!request_candidate_api_formats("openai:embedding", false).contains(&"openai:chat"));
|
||||
@@ -479,6 +488,7 @@ mod tests {
|
||||
"openai:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -488,6 +498,17 @@ mod tests {
|
||||
"openai:embedding",
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
request_candidate_api_formats("aliyun:multimodal_embedding", false),
|
||||
vec![
|
||||
"aliyun:multimodal_embedding",
|
||||
"openai:embedding",
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
]
|
||||
);
|
||||
|
||||
@@ -496,6 +517,7 @@ mod tests {
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
];
|
||||
for client_api_format in embedding_formats {
|
||||
for provider_api_format in embedding_formats {
|
||||
@@ -520,6 +542,7 @@ mod tests {
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
];
|
||||
let standard_formats = [
|
||||
"openai:chat",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod aliyun;
|
||||
pub mod claude;
|
||||
pub mod context;
|
||||
pub mod conversion;
|
||||
|
||||
@@ -50,6 +50,10 @@ pub(crate) fn from_namespace(body_json: &Value, namespace: &str) -> Option<Canon
|
||||
.get("user")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
parameters: request
|
||||
.get("parameters")
|
||||
.and_then(Value::as_object)
|
||||
.cloned(),
|
||||
extensions: namespace_extensions(
|
||||
namespace,
|
||||
request,
|
||||
@@ -60,6 +64,7 @@ pub(crate) fn from_namespace(body_json: &Value, namespace: &str) -> Option<Canon
|
||||
"dimensions",
|
||||
"task",
|
||||
"user",
|
||||
"parameters",
|
||||
],
|
||||
),
|
||||
};
|
||||
@@ -81,6 +86,9 @@ pub(crate) fn to_openai_like(
|
||||
if embedding.input.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if matches!(&embedding.input, CanonicalEmbeddingInput::Multimodal(_)) {
|
||||
return None;
|
||||
}
|
||||
let mut output = Map::new();
|
||||
output.insert(
|
||||
"model".to_string(),
|
||||
@@ -99,6 +107,9 @@ pub(crate) fn to_openai_like(
|
||||
if let Some(value) = &embedding.user {
|
||||
output.insert("user".to_string(), Value::String(value.clone()));
|
||||
}
|
||||
if let Some(value) = &embedding.parameters {
|
||||
output.insert("parameters".to_string(), Value::Object(value.clone()));
|
||||
}
|
||||
if let Some(task) = embedding
|
||||
.task
|
||||
.as_ref()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::formats::{
|
||||
aliyun,
|
||||
claude::messages as claude_messages,
|
||||
doubao,
|
||||
gemini::{self, generate_content as gemini_generate_content},
|
||||
@@ -29,7 +30,9 @@ pub fn parse_request(
|
||||
FormatId::JinaEmbedding => jina::embedding::request::from(body, ctx),
|
||||
FormatId::OpenAiRerank => openai::rerank::request::from(body, ctx),
|
||||
FormatId::JinaRerank => jina::rerank::request::from(body, ctx),
|
||||
FormatId::GeminiEmbedding | FormatId::DoubaoEmbedding => None,
|
||||
FormatId::GeminiEmbedding
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => None,
|
||||
}
|
||||
.ok_or_else(|| FormatError::RequestParseFailed {
|
||||
format: source.as_str().to_string(),
|
||||
@@ -62,6 +65,7 @@ pub fn emit_request(
|
||||
FormatId::JinaRerank => jina::rerank::request::to(&request, ctx),
|
||||
FormatId::GeminiEmbedding => gemini::embedding::request::to(&request, ctx),
|
||||
FormatId::DoubaoEmbedding => doubao::embedding::request::to(&request, ctx),
|
||||
FormatId::AliyunMultimodalEmbedding => aliyun::embedding::request::to(&request, ctx),
|
||||
}
|
||||
.ok_or_else(|| FormatError::RequestEmitFailed {
|
||||
format: target.as_str().to_string(),
|
||||
@@ -96,7 +100,8 @@ pub fn parse_response(
|
||||
| FormatId::OpenAiRerank
|
||||
| FormatId::JinaRerank
|
||||
| FormatId::GeminiEmbedding
|
||||
| FormatId::DoubaoEmbedding => None,
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => None,
|
||||
}
|
||||
.ok_or_else(|| FormatError::ResponseParseFailed {
|
||||
format: source.as_str().to_string(),
|
||||
@@ -120,7 +125,8 @@ pub fn emit_response(
|
||||
| FormatId::OpenAiRerank
|
||||
| FormatId::JinaRerank
|
||||
| FormatId::GeminiEmbedding
|
||||
| FormatId::DoubaoEmbedding => None,
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => None,
|
||||
}
|
||||
.ok_or_else(|| FormatError::ResponseEmitFailed {
|
||||
format: target.as_str().to_string(),
|
||||
@@ -252,6 +258,119 @@ mod tests {
|
||||
assert!(doubao.get("messages").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_openai_embedding_to_aliyun_multimodal_payload_shape() {
|
||||
let body = json!({
|
||||
"model": "text-embedding-3-small",
|
||||
"input": [
|
||||
{"text": "white running shoes"},
|
||||
{"image": "https://example.com/shoe.png"},
|
||||
{"multi_images": ["https://example.com/a.png", "https://example.com/b.png"]}
|
||||
],
|
||||
"dimensions": 1024,
|
||||
"parameters": {
|
||||
"enable_fusion": true,
|
||||
"res_level": 2,
|
||||
"max_video_frames": 64
|
||||
}
|
||||
});
|
||||
|
||||
let converted = convert_request(
|
||||
"openai:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
&body,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.expect("aliyun multimodal embedding conversion should succeed");
|
||||
|
||||
assert_eq!(converted["model"], "qwen3-vl-embedding");
|
||||
assert_eq!(converted["input"]["contents"], body["input"]);
|
||||
assert_eq!(converted["parameters"]["dimension"], 1024);
|
||||
assert_eq!(converted["parameters"]["enable_fusion"], true);
|
||||
assert_eq!(converted["parameters"]["res_level"], 2);
|
||||
assert_eq!(converted["parameters"]["max_video_frames"], 64);
|
||||
assert!(converted.get("messages").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliyun_embedding_conversion_rejects_token_arrays() {
|
||||
let body = json!({
|
||||
"model": "text-embedding-3-small",
|
||||
"input": [1, 2, 3]
|
||||
});
|
||||
|
||||
assert!(convert_request(
|
||||
"openai:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
&body,
|
||||
&FormatContext::default().with_mapped_model("qwen3-vl-embedding"),
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_embedding_conversion_is_aliyun_only() {
|
||||
let body = json!({
|
||||
"model": "qwen3-vl-embedding",
|
||||
"input": [
|
||||
{"text": "white running shoes"},
|
||||
{"image": "https://example.com/shoe.png"}
|
||||
]
|
||||
});
|
||||
let ctx = FormatContext::default().with_mapped_model("qwen3-vl-embedding");
|
||||
|
||||
assert!(convert_request("openai:embedding", "openai:embedding", &body, &ctx).is_err());
|
||||
assert!(convert_request("openai:embedding", "jina:embedding", &body, &ctx).is_err());
|
||||
assert!(convert_request("openai:embedding", "gemini:embedding", &body, &ctx).is_err());
|
||||
assert!(convert_request("openai:embedding", "doubao:embedding", &body, &ctx).is_err());
|
||||
assert!(convert_request(
|
||||
"openai:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
&body,
|
||||
&ctx
|
||||
)
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_aliyun_embedding_response_to_openai_shape() {
|
||||
let body = json!({
|
||||
"output": {
|
||||
"embeddings": [
|
||||
{
|
||||
"index": 0,
|
||||
"embedding": [0.1, 0.2, 0.3],
|
||||
"type": "fused"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input_tokens": 432,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": 402,
|
||||
"text_tokens": 30
|
||||
},
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 433
|
||||
},
|
||||
"request_id": "aliyun-request-1"
|
||||
});
|
||||
|
||||
let canonical =
|
||||
crate::protocol::canonical::from_embedding_to_canonical_response(&body, "aliyun")
|
||||
.expect("aliyun embedding response should parse");
|
||||
let emitted =
|
||||
crate::protocol::canonical::canonical_to_embedding_response(&canonical, "openai")
|
||||
.expect("openai embedding response should emit");
|
||||
|
||||
assert_eq!(emitted["request_id"], "aliyun-request-1");
|
||||
assert_eq!(emitted["data"][0]["embedding"], json!([0.1, 0.2, 0.3]));
|
||||
assert_eq!(emitted["data"][0]["type"], "fused");
|
||||
assert_eq!(emitted["usage"]["prompt_tokens"], 432);
|
||||
assert_eq!(emitted["usage"]["completion_tokens"], 1);
|
||||
assert_eq!(emitted["usage"]["total_tokens"], 433);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedding_registry_keeps_gemini_and_doubao_emit_only() {
|
||||
let body = json!({
|
||||
|
||||
@@ -260,7 +260,8 @@ impl ProviderStreamParser {
|
||||
| FormatId::GeminiEmbedding
|
||||
| FormatId::JinaEmbedding
|
||||
| FormatId::JinaRerank
|
||||
| FormatId::DoubaoEmbedding => return None,
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => return None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -350,7 +351,8 @@ impl ClientStreamEmitter {
|
||||
| FormatId::GeminiEmbedding
|
||||
| FormatId::JinaEmbedding
|
||||
| FormatId::JinaRerank
|
||||
| FormatId::DoubaoEmbedding => return None,
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => return None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -414,7 +416,8 @@ fn parse_provider_error(
|
||||
| FormatId::GeminiEmbedding
|
||||
| FormatId::JinaEmbedding
|
||||
| FormatId::JinaRerank
|
||||
| FormatId::DoubaoEmbedding => None,
|
||||
| FormatId::DoubaoEmbedding
|
||||
| FormatId::AliyunMultimodalEmbedding => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -439,6 +439,7 @@ fn embedding_response_namespace_for_api_format(api_format: &str) -> Option<&'sta
|
||||
"openai:embedding" => Some("openai"),
|
||||
"jina:embedding" => Some("jina"),
|
||||
"gemini:embedding" => Some("gemini"),
|
||||
"aliyun:multimodal_embedding" => Some("aliyun"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,9 +43,9 @@ pub use protocol::canonical::{
|
||||
from_gemini_to_canonical_response, from_openai_chat_to_canonical_request,
|
||||
from_openai_chat_to_canonical_response, from_openai_responses_to_canonical_request,
|
||||
from_openai_responses_to_canonical_response, CanonicalContentBlock, CanonicalEmbedding,
|
||||
CanonicalEmbeddingInput, CanonicalEmbeddingRequest, CanonicalEmbeddingResponse,
|
||||
CanonicalGenerationConfig, CanonicalInstruction, CanonicalMessage, CanonicalRequest,
|
||||
CanonicalResponse, CanonicalResponseFormat, CanonicalResponseOutput, CanonicalRole,
|
||||
CanonicalStopReason, CanonicalStreamEvent, CanonicalStreamFrame, CanonicalThinkingConfig,
|
||||
CanonicalToolChoice, CanonicalToolDefinition, CanonicalUsage,
|
||||
CanonicalEmbeddingContent, CanonicalEmbeddingInput, CanonicalEmbeddingRequest,
|
||||
CanonicalEmbeddingResponse, CanonicalGenerationConfig, CanonicalInstruction, CanonicalMessage,
|
||||
CanonicalRequest, CanonicalResponse, CanonicalResponseFormat, CanonicalResponseOutput,
|
||||
CanonicalRole, CanonicalStopReason, CanonicalStreamEvent, CanonicalStreamFrame,
|
||||
CanonicalThinkingConfig, CanonicalToolChoice, CanonicalToolDefinition, CanonicalUsage,
|
||||
};
|
||||
|
||||
@@ -246,6 +246,19 @@ pub enum CanonicalEmbeddingInput {
|
||||
StringArray(Vec<String>),
|
||||
TokenArray(Vec<i64>),
|
||||
TokenArrayArray(Vec<Vec<i64>>),
|
||||
Multimodal(Vec<CanonicalEmbeddingContent>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CanonicalEmbeddingContent {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub video: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub multi_images: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl CanonicalEmbeddingInput {
|
||||
@@ -257,6 +270,9 @@ impl CanonicalEmbeddingInput {
|
||||
}
|
||||
Self::TokenArray(values) => values.is_empty(),
|
||||
Self::TokenArrayArray(values) => values.is_empty() || values.iter().any(Vec::is_empty),
|
||||
Self::Multimodal(values) => {
|
||||
values.is_empty() || values.iter().any(CanonicalEmbeddingContent::is_empty)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,11 +280,47 @@ impl CanonicalEmbeddingInput {
|
||||
match self {
|
||||
Self::String(value) => Some(vec![value.as_str()]),
|
||||
Self::StringArray(values) => Some(values.iter().map(String::as_str).collect()),
|
||||
Self::TokenArray(_) | Self::TokenArrayArray(_) => None,
|
||||
Self::TokenArray(_) | Self::TokenArrayArray(_) | Self::Multimodal(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CanonicalEmbeddingContent {
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
let text_empty = self
|
||||
.text
|
||||
.as_ref()
|
||||
.is_some_and(|value| value.trim().is_empty());
|
||||
let image_empty = self
|
||||
.image
|
||||
.as_ref()
|
||||
.is_some_and(|value| value.trim().is_empty());
|
||||
let video_empty = self
|
||||
.video
|
||||
.as_ref()
|
||||
.is_some_and(|value| value.trim().is_empty());
|
||||
let multi_images_empty = self.multi_images.as_ref().is_some_and(|values| {
|
||||
values.is_empty() || values.iter().any(|value| value.trim().is_empty())
|
||||
});
|
||||
let has_any = self
|
||||
.text
|
||||
.as_ref()
|
||||
.is_some_and(|value| !value.trim().is_empty())
|
||||
|| self
|
||||
.image
|
||||
.as_ref()
|
||||
.is_some_and(|value| !value.trim().is_empty())
|
||||
|| self
|
||||
.video
|
||||
.as_ref()
|
||||
.is_some_and(|value| !value.trim().is_empty())
|
||||
|| self.multi_images.as_ref().is_some_and(|values| {
|
||||
!values.is_empty() && values.iter().all(|value| !value.trim().is_empty())
|
||||
});
|
||||
!has_any || text_empty || image_empty || video_empty || multi_images_empty
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CanonicalEmbeddingRequest {
|
||||
pub input: CanonicalEmbeddingInput,
|
||||
@@ -280,6 +332,8 @@ pub struct CanonicalEmbeddingRequest {
|
||||
pub task: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parameters: Option<Map<String, Value>>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub extensions: BTreeMap<String, Value>,
|
||||
}
|
||||
@@ -502,6 +556,7 @@ pub(crate) fn canonical_to_embedding_request(
|
||||
"jina" => crate::formats::jina::embedding::request::to(canonical, &ctx),
|
||||
"gemini" => crate::formats::gemini::embedding::request::to(canonical, &ctx),
|
||||
"doubao" => crate::formats::doubao::embedding::request::to(canonical, &ctx),
|
||||
"aliyun" => crate::formats::aliyun::embedding::request::to(canonical, &ctx),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -699,6 +754,7 @@ pub fn from_embedding_to_canonical_response(
|
||||
}
|
||||
"jina" => crate::formats::openai::embedding::response::from_namespace(body_json, "jina"),
|
||||
"gemini" => crate::formats::gemini::embedding::response::from(body_json),
|
||||
"aliyun" => crate::formats::aliyun::embedding::response::from(body_json),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -5244,8 +5300,8 @@ mod tests {
|
||||
from_gemini_to_canonical_request, from_gemini_to_canonical_response,
|
||||
from_openai_chat_to_canonical_request, from_openai_chat_to_canonical_response,
|
||||
from_openai_responses_to_canonical_request, from_openai_responses_to_canonical_response,
|
||||
CanonicalContentBlock, CanonicalEmbedding, CanonicalEmbeddingInput,
|
||||
CanonicalEmbeddingRequest, CanonicalRole, CanonicalUsage,
|
||||
CanonicalContentBlock, CanonicalEmbedding, CanonicalEmbeddingContent,
|
||||
CanonicalEmbeddingInput, CanonicalEmbeddingRequest, CanonicalRole, CanonicalUsage,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
@@ -5301,6 +5357,44 @@ mod tests {
|
||||
"nested token array",
|
||||
CanonicalEmbeddingInput::TokenArrayArray(vec![vec![1, 2], vec![3, 4]]),
|
||||
),
|
||||
(
|
||||
json!([
|
||||
{"text": "white running shoes"},
|
||||
{"image": "https://example.com/shoe.png"},
|
||||
{"video": "https://example.com/demo.mp4"},
|
||||
{"multi_images": ["https://example.com/a.png", "https://example.com/b.png"]}
|
||||
]),
|
||||
"multimodal array",
|
||||
CanonicalEmbeddingInput::Multimodal(vec![
|
||||
CanonicalEmbeddingContent {
|
||||
text: Some("white running shoes".to_string()),
|
||||
image: None,
|
||||
video: None,
|
||||
multi_images: None,
|
||||
},
|
||||
CanonicalEmbeddingContent {
|
||||
text: None,
|
||||
image: Some("https://example.com/shoe.png".to_string()),
|
||||
video: None,
|
||||
multi_images: None,
|
||||
},
|
||||
CanonicalEmbeddingContent {
|
||||
text: None,
|
||||
image: None,
|
||||
video: Some("https://example.com/demo.mp4".to_string()),
|
||||
multi_images: None,
|
||||
},
|
||||
CanonicalEmbeddingContent {
|
||||
text: None,
|
||||
image: None,
|
||||
video: None,
|
||||
multi_images: Some(vec![
|
||||
"https://example.com/a.png".to_string(),
|
||||
"https://example.com/b.png".to_string(),
|
||||
]),
|
||||
},
|
||||
]),
|
||||
),
|
||||
];
|
||||
|
||||
for (input, label, expected_input) in cases {
|
||||
@@ -5327,6 +5421,9 @@ mod tests {
|
||||
json!({"model": "text-embedding-3-small", "input": []}),
|
||||
json!({"model": "text-embedding-3-small", "input": [1, "two"]}),
|
||||
json!({"model": "text-embedding-3-small", "input": [[1], []]}),
|
||||
json!({"model": "text-embedding-3-small", "input": [{"image": " "}]}),
|
||||
json!({"model": "text-embedding-3-small", "input": [{"multi_images": []}]}),
|
||||
json!({"model": "text-embedding-3-small", "input": ["hello", {"image": "https://example.com/a.png"}]}),
|
||||
json!({"model": "", "input": "hello"}),
|
||||
json!({"input": "hello"}),
|
||||
json!({"model": "text-embedding-3-small", "messages": []}),
|
||||
@@ -5396,6 +5493,7 @@ mod tests {
|
||||
dimensions: Some(2),
|
||||
task: None,
|
||||
user: None,
|
||||
parameters: None,
|
||||
extensions: Default::default(),
|
||||
}),
|
||||
..Default::default()
|
||||
@@ -5441,6 +5539,7 @@ mod tests {
|
||||
dimensions: Some(1536),
|
||||
task: Some("retrieval.passage".to_string()),
|
||||
user: Some("user-1".to_string()),
|
||||
parameters: None,
|
||||
extensions: Default::default(),
|
||||
}),
|
||||
..Default::default()
|
||||
@@ -5493,6 +5592,7 @@ mod tests {
|
||||
dimensions: None,
|
||||
task: None,
|
||||
user: None,
|
||||
parameters: None,
|
||||
extensions: Default::default(),
|
||||
}),
|
||||
..Default::default()
|
||||
|
||||
@@ -7,6 +7,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
"/v1/embeddings",
|
||||
"/jina/v1/embeddings",
|
||||
];
|
||||
|
||||
@@ -25,6 +25,7 @@ const EMBEDDING_API_FORMATS: &[&str] = &[
|
||||
"gemini:embedding",
|
||||
"jina:embedding",
|
||||
"doubao:embedding",
|
||||
"aliyun:multimodal_embedding",
|
||||
"/v1/embeddings",
|
||||
"/jina/v1/embeddings",
|
||||
];
|
||||
|
||||
@@ -71,6 +71,7 @@ SELECT
|
||||
OR COALESCE(gm.config->'api_formats' @> '["jina:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(gm.config->'api_formats' @> '["gemini:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(gm.config->'api_formats' @> '["doubao:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(gm.config->'api_formats' @> '["aliyun:multimodal_embedding"]'::jsonb, FALSE)
|
||||
OR LOWER(COALESCE(m.config->>'embedding', 'false')) = 'true'
|
||||
OR LOWER(COALESCE(m.config->>'model_type', '')) = 'embedding'
|
||||
OR LOWER(COALESCE(m.config->>'type', '')) = 'embedding'
|
||||
@@ -80,6 +81,7 @@ SELECT
|
||||
OR COALESCE(m.config::jsonb->'api_formats' @> '["jina:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(m.config::jsonb->'api_formats' @> '["gemini:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(m.config::jsonb->'api_formats' @> '["doubao:embedding"]'::jsonb, FALSE)
|
||||
OR COALESCE(m.config::jsonb->'api_formats' @> '["aliyun:multimodal_embedding"]'::jsonb, FALSE)
|
||||
) AS supports_embedding,
|
||||
m.is_active
|
||||
FROM models m
|
||||
|
||||
@@ -184,7 +184,11 @@ pub fn request_pair_transport_unsupported_reason(
|
||||
)
|
||||
}
|
||||
}
|
||||
"openai:embedding" | "jina:embedding" | "doubao:embedding" | "openai:rerank"
|
||||
"openai:embedding"
|
||||
| "jina:embedding"
|
||||
| "doubao:embedding"
|
||||
| "aliyun:multimodal_embedding"
|
||||
| "openai:rerank"
|
||||
| "jina:rerank" => local_standard_transport_unsupported_reason_with_network(
|
||||
transport,
|
||||
provider_api_format.as_str(),
|
||||
@@ -221,6 +225,7 @@ fn request_direct_auth_for_provider_format(
|
||||
| "openai:embedding"
|
||||
| "jina:embedding"
|
||||
| "doubao:embedding"
|
||||
| "aliyun:multimodal_embedding"
|
||||
| "openai:rerank"
|
||||
| "jina:rerank" => resolve_local_openai_bearer_auth(transport),
|
||||
"gemini:generate_content" | "gemini:embedding" => {
|
||||
|
||||
@@ -225,7 +225,7 @@ fn endpoint_kind_allows_embedding(endpoint_kind: Option<&str>) -> bool {
|
||||
.map(|value| {
|
||||
matches!(
|
||||
value.to_ascii_lowercase().as_str(),
|
||||
"embedding" | "embeddings"
|
||||
"embedding" | "embeddings" | "multimodal_embedding" | "multimodal_embeddings"
|
||||
)
|
||||
})
|
||||
.unwrap_or(true)
|
||||
@@ -349,10 +349,13 @@ mod tests {
|
||||
("jina", "jina:embedding"),
|
||||
("doubao", "doubao:embedding"),
|
||||
("volcengine", "doubao:embedding"),
|
||||
("aliyun", "aliyun:multimodal_embedding"),
|
||||
("dashscope", "aliyun:multimodal_embedding"),
|
||||
("custom", "openai:embedding"),
|
||||
("custom", "gemini:embedding"),
|
||||
("custom", "jina:embedding"),
|
||||
("custom", "doubao:embedding"),
|
||||
("custom", "aliyun:multimodal_embedding"),
|
||||
] {
|
||||
let transport = sample_transport(provider_type, api_format, Some("embedding"));
|
||||
assert_eq!(
|
||||
|
||||
@@ -84,6 +84,7 @@ pub enum ProviderLocalEmbeddingSupport {
|
||||
Gemini,
|
||||
Jina,
|
||||
Doubao,
|
||||
Aliyun,
|
||||
}
|
||||
|
||||
impl ProviderLocalEmbeddingSupport {
|
||||
@@ -99,11 +100,13 @@ impl ProviderLocalEmbeddingSupport {
|
||||
| "jina:embedding"
|
||||
| "jina:rerank"
|
||||
| "doubao:embedding"
|
||||
| "aliyun:multimodal_embedding"
|
||||
),
|
||||
Self::OpenAi => matches!(api_format.as_str(), "openai:embedding" | "openai:rerank"),
|
||||
Self::Gemini => api_format == "gemini:embedding",
|
||||
Self::Jina => matches!(api_format.as_str(), "jina:embedding" | "jina:rerank"),
|
||||
Self::Doubao => api_format == "doubao:embedding",
|
||||
Self::Aliyun => api_format == "aliyun:multimodal_embedding",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -192,6 +195,10 @@ const DOUBAO_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
|
||||
local_embedding_support: ProviderLocalEmbeddingSupport::Doubao,
|
||||
..STANDARD_RUNTIME_POLICY
|
||||
};
|
||||
const ALIYUN_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
|
||||
local_embedding_support: ProviderLocalEmbeddingSupport::Aliyun,
|
||||
..STANDARD_RUNTIME_POLICY
|
||||
};
|
||||
|
||||
const CLAUDE_CODE_RUNTIME_POLICY: ProviderRuntimePolicy = ProviderRuntimePolicy {
|
||||
fixed_provider: true,
|
||||
@@ -470,6 +477,7 @@ pub fn provider_runtime_policy(provider_type: &str) -> ProviderRuntimePolicy {
|
||||
"gemini" | "google" => GEMINI_RUNTIME_POLICY,
|
||||
"jina" => JINA_RUNTIME_POLICY,
|
||||
"doubao" | "volcengine" => DOUBAO_RUNTIME_POLICY,
|
||||
"aliyun" | "dashscope" => ALIYUN_RUNTIME_POLICY,
|
||||
_ => STANDARD_RUNTIME_POLICY,
|
||||
}
|
||||
}
|
||||
@@ -875,6 +883,8 @@ mod tests {
|
||||
("jina", "jina:embedding"),
|
||||
("doubao", "doubao:embedding"),
|
||||
("volcengine", "doubao:embedding"),
|
||||
("aliyun", "aliyun:multimodal_embedding"),
|
||||
("dashscope", "aliyun:multimodal_embedding"),
|
||||
] {
|
||||
assert!(
|
||||
provider_type_supports_local_embedding_transport(provider_type, api_format),
|
||||
@@ -888,6 +898,8 @@ mod tests {
|
||||
("vertex_ai", "openai:embedding"),
|
||||
("jina", "doubao:embedding"),
|
||||
("doubao", "jina:embedding"),
|
||||
("aliyun", "openai:embedding"),
|
||||
("openai", "aliyun:multimodal_embedding"),
|
||||
("claude_code", "openai:embedding"),
|
||||
("openai", "openai:chat"),
|
||||
] {
|
||||
|
||||
@@ -127,6 +127,10 @@ fn build_transport_request_url_inner(
|
||||
"openai:embedding" | "jina:embedding" => {
|
||||
build_provider_embedding_v1_url(&transport.endpoint.base_url, params.request_query)
|
||||
}
|
||||
"aliyun:multimodal_embedding" => build_aliyun_multimodal_embedding_url(
|
||||
&transport.endpoint.base_url,
|
||||
params.request_query,
|
||||
),
|
||||
"openai:rerank" | "jina:rerank" => {
|
||||
build_provider_rerank_v1_url(&transport.endpoint.base_url, params.request_query)
|
||||
}
|
||||
@@ -425,6 +429,18 @@ fn build_provider_embedding_v1_url(upstream_base_url: &str, query: Option<&str>)
|
||||
build_provider_api_root_url(upstream_base_url, "/embeddings", query)
|
||||
}
|
||||
|
||||
fn build_aliyun_multimodal_embedding_url(
|
||||
upstream_base_url: &str,
|
||||
query: Option<&str>,
|
||||
) -> Option<String> {
|
||||
build_passthrough_path_url(
|
||||
upstream_base_url,
|
||||
"/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding",
|
||||
query,
|
||||
&[],
|
||||
)
|
||||
}
|
||||
|
||||
fn build_provider_rerank_v1_url(upstream_base_url: &str, query: Option<&str>) -> Option<String> {
|
||||
build_provider_api_root_url(upstream_base_url, "/rerank", query)
|
||||
}
|
||||
@@ -1019,6 +1035,12 @@ mod tests {
|
||||
"https://ark.volces.example/api/v3",
|
||||
None,
|
||||
);
|
||||
let aliyun = sample_transport(
|
||||
"aliyun",
|
||||
"aliyun:multimodal_embedding",
|
||||
"https://dashscope.aliyuncs.com",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
build_transport_request_url(
|
||||
@@ -1078,6 +1100,20 @@ mod tests {
|
||||
.as_deref(),
|
||||
Some("https://ark.volces.example/api/v3/embeddings")
|
||||
);
|
||||
assert_eq!(
|
||||
build_transport_request_url(
|
||||
&aliyun,
|
||||
TransportRequestUrlParams {
|
||||
provider_api_format: "aliyun:multimodal_embedding",
|
||||
mapped_model: Some("qwen3-vl-embedding"),
|
||||
upstream_is_stream: false,
|
||||
request_query: None,
|
||||
kiro_api_region: None,
|
||||
},
|
||||
)
|
||||
.as_deref(),
|
||||
Some("https://dashscope.aliyuncs.com/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -169,6 +169,14 @@ pub fn build_same_format_provider_request_body(
|
||||
);
|
||||
}
|
||||
|
||||
if embedding_multimodal_input_requires_aliyun_provider(
|
||||
input.client_api_format,
|
||||
input.provider_api_format,
|
||||
input.body_json,
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut provider_request_body = if aether_ai_formats::api_format_alias_matches(
|
||||
input.client_api_format,
|
||||
input.provider_api_format,
|
||||
@@ -245,6 +253,31 @@ pub fn build_same_format_provider_request_body(
|
||||
Some(provider_request_body)
|
||||
}
|
||||
|
||||
fn embedding_multimodal_input_requires_aliyun_provider(
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
body_json: &Value,
|
||||
) -> bool {
|
||||
aether_ai_formats::is_embedding_api_format(client_api_format)
|
||||
&& embedding_input_is_multimodal(body_json.get("input"))
|
||||
&& aether_ai_formats::normalize_api_format_alias(provider_api_format)
|
||||
!= "aliyun:multimodal_embedding"
|
||||
}
|
||||
|
||||
fn embedding_input_is_multimodal(value: Option<&Value>) -> bool {
|
||||
value
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|items| !items.is_empty() && items.iter().all(embedding_content_is_multimodal))
|
||||
}
|
||||
|
||||
fn embedding_content_is_multimodal(value: &Value) -> bool {
|
||||
value.as_object().is_some_and(|object| {
|
||||
["text", "image", "video", "multi_images"]
|
||||
.iter()
|
||||
.any(|key| object.contains_key(*key))
|
||||
})
|
||||
}
|
||||
|
||||
fn strip_gemini_function_response_ids(value: &mut Value) {
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
@@ -849,6 +882,33 @@ mod tests {
|
||||
assert_eq!(body.get("stream"), Some(&json!(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_format_embedding_body_rejects_multimodal_for_openai_like_provider() {
|
||||
let body = build_same_format_provider_request_body(SameFormatProviderRequestBodyInput {
|
||||
body_json: &json!({
|
||||
"model": "qwen3-vl-embedding",
|
||||
"input": [
|
||||
{"text": "white running shoes"},
|
||||
{"image": "https://example.com/shoe.png"}
|
||||
]
|
||||
}),
|
||||
mapped_model: "openai-qwen-fallback",
|
||||
client_api_format: "openai:embedding",
|
||||
provider_api_format: "openai:embedding",
|
||||
source_model: Some("qwen3-vl-embedding"),
|
||||
family: SameFormatProviderFamily::Standard,
|
||||
body_rules: None,
|
||||
request_headers: None,
|
||||
upstream_is_stream: false,
|
||||
force_body_stream_field: false,
|
||||
kiro_auth_config: None,
|
||||
is_claude_code: false,
|
||||
enable_model_directives: false,
|
||||
});
|
||||
|
||||
assert!(body.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_format_standard_body_overrides_client_stream_for_non_stream_upstream() {
|
||||
let body = build_same_format_provider_request_body(SameFormatProviderRequestBodyInput {
|
||||
|
||||
Reference in New Issue
Block a user