Merge pull request #1024 from sanders41/rest-api-embedder

Update rest embedder for Meilisearch v1.10.0
sanders41 · Jul 30, 2024 · e635caa · e635caa
2 parents 2b48328 + bdebdc6
commit e635caa
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 57 deletions.
diff --git a/meilisearch_python_sdk/models/settings.py b/meilisearch_python_sdk/models/settings.py
@@ -78,12 +78,9 @@ class RestEmbedder(CamelBase):
     api_key: str | None = None
     dimensions: int
     document_template: str | None = None
-    input_field: list[str] | None = None
-    input_type: str = "text"
-    query: JsonDict = {}
-    path_to_embeddings: list[str] | None = None
-    embedding_object: list[str] | None = None
     distribution: Distribution | None = None
+    request: JsonDict
+    response: JsonDict
 
 
 class UserProvidedEmbedder(CamelBase):

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -282,13 +282,5 @@ def new_settings():
         non_separator_tokens=["#", "@"],
         search_cutoff_ms=100,
         dictionary=["S.O", "S.O.S"],
-        # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585
-        # embedders={
-        #     "default": UserProvidedEmbedder(dimensions=512),
-        #     # "test1": HuggingFaceEmbedder(),
-        #     "test2": OpenAiEmbedder(),
-        #     # "test3": OllamaEmbedder(model="nomic-embed-text"),
-        #     "test4": RestEmbedder(url="https://myurl.com"),
-        # },
         proximity_precision=ProximityPrecision.BY_ATTRIBUTE,
     )
diff --git a/tests/test_async_index.py b/tests/test_async_index.py
@@ -165,12 +165,6 @@ async def test_update_settings(compress, async_empty_index, new_settings):
     assert response.non_separator_tokens == new_settings.non_separator_tokens
     assert response.search_cutoff_ms == new_settings.search_cutoff_ms
     assert response.dictionary == new_settings.dictionary
-    # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585
-    # assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
-    # assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
-    # assert response.embedders["test4"].source == "rest"
 
 
 async def test_reset_settings(async_empty_index, new_settings, default_ranking_rules):
@@ -189,12 +183,6 @@ async def test_reset_settings(async_empty_index, new_settings, default_ranking_r
     assert response.typo_tolerance.enabled is False
     assert response.pagination == new_settings.pagination
     assert response.proximity_precision == new_settings.proximity_precision
-    # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585
-    # assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
-    # assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
-    # assert response.embedders["test4"].source == "rest"
     response = await index.reset_settings()
     update = await async_wait_for_task(index.http_client, response.task_uid)
     assert update.status == "succeeded"
@@ -694,31 +682,35 @@ async def test_update_embedders(async_empty_index):
     embedders = Embedders(
         embedders={
             "default": UserProvidedEmbedder(dimensions=512),
-            # "test1": HuggingFaceEmbedder(),
             "test2": OpenAiEmbedder(),
-            # "test3": OllamaEmbedder(model="nomic-embed-text"),
-            "test4": RestEmbedder(url="https://myurl.com", dimensions=512),
+            "test4": RestEmbedder(
+                url="https://myurl.com",
+                dimensions=512,
+                request={"request": {"model": "minillm", "prompt": "{{text}}"}},
+                response={"response": {"embedding": "{{embedding}}"}},
+            ),
         }
     )
     index = await async_empty_index()
     response = await index.update_embedders(embedders)
     await async_wait_for_task(index.http_client, response.task_uid)
     response = await index.get_embedders()
     assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
     assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
     assert response.embedders["test4"].source == "rest"
 
 
 async def test_reset_embedders(async_empty_index):
     embedders = Embedders(
         embedders={
             "default": UserProvidedEmbedder(dimensions=512),
-            # "test1": HuggingFaceEmbedder(),
             "test2": OpenAiEmbedder(),
-            # "test3": OllamaEmbedder(model="nomic-embed-text"),
-            "test4": RestEmbedder(url="https://myurl.com", dimensions=512),
+            "test4": RestEmbedder(
+                url="https://myurl.com",
+                dimensions=512,
+                request={"request": {"model": "minillm", "prompt": "{{text}}"}},
+                response={"response": {"embedding": "{{embedding}}"}},
+            ),
         }
     )
     index = await async_empty_index()
@@ -727,9 +719,7 @@ async def test_reset_embedders(async_empty_index):
     assert update.status == "succeeded"
     response = await index.get_embedders()
     assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
     assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
     assert response.embedders["test4"].source == "rest"
     response = await index.reset_embedders()
     await async_wait_for_task(index.http_client, response.task_uid)

diff --git a/tests/test_index.py b/tests/test_index.py
@@ -157,12 +157,6 @@ def test_update_settings(compress, empty_index, new_settings):
     assert response.non_separator_tokens == new_settings.non_separator_tokens
     assert response.search_cutoff_ms == new_settings.search_cutoff_ms
     assert response.dictionary == new_settings.dictionary
-    # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585
-    # assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
-    # assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
-    # assert response.embedders["test4"].source == "rest"
 
 
 def test_reset_settings(empty_index, new_settings, default_ranking_rules):
@@ -181,12 +175,6 @@ def test_reset_settings(empty_index, new_settings, default_ranking_rules):
     assert response.typo_tolerance.enabled is False
     assert response.pagination == new_settings.pagination
     assert response.proximity_precision == new_settings.proximity_precision
-    # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585
-    # assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
-    # assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
-    # assert response.embedders["test4"].source == "rest"
     response = index.reset_settings()
     update = wait_for_task(index.http_client, response.task_uid)
     assert update.status == "succeeded"
@@ -679,31 +667,35 @@ def test_update_embedders(empty_index):
     embedders = Embedders(
         embedders={
             "default": UserProvidedEmbedder(dimensions=512),
-            # "test1": HuggingFaceEmbedder(),
             "test2": OpenAiEmbedder(),
-            # "test3": OllamaEmbedder(model="nomic-embed-text"),
-            "test4": RestEmbedder(url="https://myurl.com", dimensions=512),
+            "test4": RestEmbedder(
+                url="https://myurl.com",
+                dimensions=512,
+                request={"request": {"model": "minillm", "prompt": "{{text}}"}},
+                response={"response": {"embedding": "{{embedding}}"}},
+            ),
         }
     )
     index = empty_index()
     response = index.update_embedders(embedders)
     wait_for_task(index.http_client, response.task_uid)
     response = index.get_embedders()
     assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
     assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
     assert response.embedders["test4"].source == "rest"
 
 
 def test_reset_embedders(empty_index):
     embedders = Embedders(
         embedders={
             "default": UserProvidedEmbedder(dimensions=512),
-            # "test1": HuggingFaceEmbedder(),
             "test2": OpenAiEmbedder(),
-            # "test3": OllamaEmbedder(model="some_model"),
-            "test4": RestEmbedder(url="https://myurl.com", dimensions=512),
+            "test4": RestEmbedder(
+                url="https://myurl.com",
+                dimensions=512,
+                request={"request": {"model": "minillm", "prompt": "{{text}}"}},
+                response={"response": {"embedding": "{{embedding}}"}},
+            ),
         }
     )
     index = empty_index()
@@ -712,9 +704,7 @@ def test_reset_embedders(empty_index):
     assert update.status == "succeeded"
     response = index.get_embedders()
     assert response.embedders["default"].source == "userProvided"
-    # assert response.embedders["test1"].source == "huggingFace"
     assert response.embedders["test2"].source == "openAi"
-    # assert response.embedders["test3"].source == "ollama"
     assert response.embedders["test4"].source == "rest"
     response = index.reset_embedders()
     wait_for_task(index.http_client, response.task_uid)