Skip to content

Commit f72885d

Browse files
kumarUjjawalkumarUjjawal
committed
Add support for multi-modal search
1 parent 910f9e9 commit f72885d

File tree

4 files changed

+166
-0
lines changed

4 files changed

+166
-0
lines changed

‎.code-samples.meilisearch.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1941,6 +1941,20 @@ search_parameter_reference_retrieve_vectors_1: |-
19411941
.execute()
19421942
.await
19431943
.unwrap();
1944+
search_parameter_reference_media_1: |-
1945+
let results = index
1946+
.search()
1947+
.with_hybrid("EMBEDDER_NAME", 0.5)
1948+
.with_media(json!({
1949+
"FIELD_A": "VALUE_A",
1950+
"FIELD_B": {
1951+
"FIELD_C": "VALUE_B",
1952+
"FIELD_D": "VALUE_C"
1953+
}
1954+
}))
1955+
.execute()
1956+
.await
1957+
.unwrap();
19441958
update_embedders_1: |-
19451959
let embedders = HashMap::from([(
19461960
String::from("default"),

‎src/features.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ pub struct ExperimentalFeaturesResult {
1414
pub contains_filter: bool,
1515
pub network: bool,
1616
pub edit_documents_by_function: bool,
17+
#[serde(default)]
18+
pub multimodal: bool,
1719
}
1820

1921
/// Struct representing the experimental features request.
@@ -45,6 +47,8 @@ pub struct ExperimentalFeatures<'a, Http: HttpClient> {
4547
pub network: Option<bool>,
4648
#[serde(skip_serializing_if = "Option::is_none")]
4749
pub edit_documents_by_function: Option<bool>,
50+
#[serde(skip_serializing_if = "Option::is_none")]
51+
pub multimodal: Option<bool>,
4852
}
4953

5054
impl<'a, Http: HttpClient> ExperimentalFeatures<'a, Http> {
@@ -57,6 +61,7 @@ impl<'a, Http: HttpClient> ExperimentalFeatures<'a, Http> {
5761
network: None,
5862
contains_filter: None,
5963
edit_documents_by_function: None,
64+
multimodal: None,
6065
}
6166
}
6267

@@ -140,6 +145,11 @@ impl<'a, Http: HttpClient> ExperimentalFeatures<'a, Http> {
140145
self.network = Some(network);
141146
self
142147
}
148+
149+
pub fn set_multimodal(&mut self, multimodal: bool) -> &mut Self {
150+
self.multimodal = Some(multimodal);
151+
self
152+
}
143153
}
144154

145155
#[cfg(test)]
@@ -155,6 +165,7 @@ mod tests {
155165
features.set_contains_filter(true);
156166
features.set_network(true);
157167
features.set_edit_documents_by_function(true);
168+
features.set_multimodal(true);
158169
let _ = features.update().await.unwrap();
159170

160171
let res = features.get().await.unwrap();
@@ -163,5 +174,6 @@ mod tests {
163174
assert!(res.contains_filter);
164175
assert!(res.network);
165176
assert!(res.edit_documents_by_function);
177+
assert!(res.multimodal);
166178
}
167179
}

‎src/search.rs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,10 @@ pub struct SearchQuery<'a, Http: HttpClient> {
408408
#[serde(skip_serializing_if = "Option::is_none")]
409409
pub retrieve_vectors: Option<bool>,
410410

411+
/// Provides multimodal data for search queries.
412+
#[serde(skip_serializing_if = "Option::is_none")]
413+
pub media: Option<Value>,
414+
411415
#[serde(skip_serializing_if = "Option::is_none")]
412416
pub(crate) federation_options: Option<QueryFederationOptions>,
413417
}
@@ -449,6 +453,7 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
449453
hybrid: None,
450454
vector: None,
451455
retrieve_vectors: None,
456+
media: None,
452457
distinct: None,
453458
ranking_score_threshold: None,
454459
locales: None,
@@ -695,6 +700,12 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
695700
self
696701
}
697702

703+
/// Attach media fragments to the search query.
704+
pub fn with_media<'b>(&'b mut self, media: Value) -> &'b mut SearchQuery<'a, Http> {
705+
self.media = Some(media);
706+
self
707+
}
708+
698709
pub fn with_distinct<'b>(&'b mut self, distinct: &'a str) -> &'b mut SearchQuery<'a, Http> {
699710
self.distinct = Some(distinct);
700711
self
@@ -1096,6 +1107,34 @@ pub(crate) mod tests {
10961107
use serde::{Deserialize, Serialize};
10971108
use serde_json::{json, Map, Value};
10981109

1110+
#[test]
1111+
fn search_query_serializes_media_parameter() {
1112+
let client = Client::new("http://localhost:7700", Some("masterKey")).unwrap();
1113+
let index = client.index("media_query");
1114+
let mut query = SearchQuery::new(&index);
1115+
1116+
query.with_query("example").with_media(json!({
1117+
"FIELD_A": "VALUE_A",
1118+
"FIELD_B": {
1119+
"FIELD_C": "VALUE_B",
1120+
"FIELD_D": "VALUE_C"
1121+
}
1122+
}));
1123+
1124+
let serialized = serde_json::to_value(&query.build()).unwrap();
1125+
1126+
assert_eq!(
1127+
serialized.get("media"),
1128+
Some(&json!({
1129+
"FIELD_A": "VALUE_A",
1130+
"FIELD_B": {
1131+
"FIELD_C": "VALUE_B",
1132+
"FIELD_D": "VALUE_C"
1133+
}
1134+
}))
1135+
);
1136+
}
1137+
10991138
#[derive(Debug, Serialize, Deserialize, PartialEq)]
11001139
pub struct Nested {
11011140
child: String,

‎src/settings.rs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,20 @@ pub struct Embedder {
146146
/// Configures embedder to vectorize search queries (composite embedders only)
147147
#[serde(skip_serializing_if = "Option::is_none")]
148148
pub search_embedder: Option<Box<Embedder>>,
149+
150+
/// Configures multimodal embedding generation at indexing time.
151+
#[serde(skip_serializing_if = "Option::is_none")]
152+
pub indexing_fragments: Option<HashMap<String, EmbedderFragment>>,
153+
154+
/// Configures incoming media fragments for multimodal search queries.
155+
#[serde(skip_serializing_if = "Option::is_none")]
156+
pub search_fragments: Option<HashMap<String, EmbedderFragment>>,
157+
}
158+
159+
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)]
160+
#[serde(rename_all = "camelCase")]
161+
pub struct EmbedderFragment {
162+
pub value: serde_json::Value,
149163
}
150164

151165
#[derive(Serialize, Deserialize, Default, Debug, Clone, Eq, PartialEq)]
@@ -2798,6 +2812,7 @@ mod tests {
27982812

27992813
use crate::client::*;
28002814
use meilisearch_test_macro::meilisearch_test;
2815+
use serde_json::json;
28012816

28022817
#[meilisearch_test]
28032818
async fn test_set_faceting_settings(client: Client, index: Index) {
@@ -3139,6 +3154,92 @@ mod tests {
31393154
assert_eq!(embedders, res);
31403155
}
31413156

3157+
#[test]
3158+
fn embedder_with_fragments_serializes() {
3159+
let embedder = Embedder {
3160+
source: EmbedderSource::Rest,
3161+
url: Some(String::from("https://example.com/embeddings")),
3162+
indexing_fragments: Some(HashMap::from([(
3163+
String::from("default"),
3164+
EmbedderFragment {
3165+
value: json!({
3166+
"content": [
3167+
{ "type": "text", "text": "{{ doc.description }}" }
3168+
]
3169+
}),
3170+
},
3171+
)])),
3172+
search_fragments: Some(HashMap::from([(
3173+
String::from("default"),
3174+
EmbedderFragment {
3175+
value: json!({
3176+
"content": [
3177+
{ "type": "text", "text": "{{ query.q }}" }
3178+
]
3179+
}),
3180+
},
3181+
)])),
3182+
request: Some(json!({
3183+
"input": [
3184+
"{{fragment}}",
3185+
"{{..}}"
3186+
],
3187+
"model": "example-model"
3188+
})),
3189+
response: Some(json!({
3190+
"data": [
3191+
{
3192+
"embedding": "{{embedding}}"
3193+
},
3194+
"{{..}}"
3195+
]
3196+
})),
3197+
..Default::default()
3198+
};
3199+
3200+
let serialized = serde_json::to_value(&embedder).unwrap();
3201+
3202+
assert_eq!(
3203+
serialized
3204+
.get("indexingFragments")
3205+
.and_then(|value| value.get("default"))
3206+
.and_then(|value| value.get("value"))
3207+
.and_then(|value| value.get("content"))
3208+
.and_then(|value| value.get(0))
3209+
.and_then(|value| value.get("text")),
3210+
Some(&json!("{{ doc.description }}"))
3211+
);
3212+
3213+
assert_eq!(
3214+
serialized
3215+
.get("searchFragments")
3216+
.and_then(|value| value.get("default"))
3217+
.and_then(|value| value.get("value"))
3218+
.and_then(|value| value.get("content"))
3219+
.and_then(|value| value.get(0))
3220+
.and_then(|value| value.get("text")),
3221+
Some(&json!("{{ query.q }}"))
3222+
);
3223+
3224+
assert_eq!(
3225+
serialized.get("request"),
3226+
Some(&json!({
3227+
"input": ["{{fragment}}", "{{..}}"],
3228+
"model": "example-model"
3229+
}))
3230+
);
3231+
3232+
assert_eq!(
3233+
serialized.get("response"),
3234+
Some(&json!({
3235+
"data": [
3236+
{ "embedding": "{{embedding}}" },
3237+
"{{..}}"
3238+
]
3239+
}))
3240+
);
3241+
}
3242+
31423243
#[meilisearch_test]
31433244
async fn test_reset_proximity_precision(index: Index) {
31443245
let expected = "byWord".to_string();

0 commit comments

Comments
 (0)