From bdebdc644a5d611132d910b79aabe1b59c193a89 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Mon, 29 Jul 2024 21:30:53 -0400 Subject: [PATCH] Update rest embedder for Meilisearch v1.10.0 --- meilisearch_python_sdk/models/settings.py | 7 ++--- tests/conftest.py | 8 ------ tests/test_async_index.py | 34 ++++++++--------------- tests/test_index.py | 34 ++++++++--------------- 4 files changed, 26 insertions(+), 57 deletions(-) diff --git a/meilisearch_python_sdk/models/settings.py b/meilisearch_python_sdk/models/settings.py index 5d1ee2f5..5aad2a60 100644 --- a/meilisearch_python_sdk/models/settings.py +++ b/meilisearch_python_sdk/models/settings.py @@ -78,12 +78,9 @@ class RestEmbedder(CamelBase): api_key: str | None = None dimensions: int document_template: str | None = None - input_field: list[str] | None = None - input_type: str = "text" - query: JsonDict = {} - path_to_embeddings: list[str] | None = None - embedding_object: list[str] | None = None distribution: Distribution | None = None + request: JsonDict + response: JsonDict class UserProvidedEmbedder(CamelBase): diff --git a/tests/conftest.py b/tests/conftest.py index a590228e..920c1ed2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -282,13 +282,5 @@ def new_settings(): non_separator_tokens=["#", "@"], search_cutoff_ms=100, dictionary=["S.O", "S.O.S"], - # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585 - # embedders={ - # "default": UserProvidedEmbedder(dimensions=512), - # # "test1": HuggingFaceEmbedder(), - # "test2": OpenAiEmbedder(), - # # "test3": OllamaEmbedder(model="nomic-embed-text"), - # "test4": RestEmbedder(url="https://myurl.com"), - # }, proximity_precision=ProximityPrecision.BY_ATTRIBUTE, ) diff --git a/tests/test_async_index.py b/tests/test_async_index.py index 79ed9e1c..3809c5a3 100644 --- a/tests/test_async_index.py +++ b/tests/test_async_index.py @@ -165,12 +165,6 @@ async def test_update_settings(compress, async_empty_index, new_settings): assert response.non_separator_tokens == new_settings.non_separator_tokens assert response.search_cutoff_ms == new_settings.search_cutoff_ms assert response.dictionary == new_settings.dictionary - # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585 - # assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" - # assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" - # assert response.embedders["test4"].source == "rest" async def test_reset_settings(async_empty_index, new_settings, default_ranking_rules): @@ -189,12 +183,6 @@ async def test_reset_settings(async_empty_index, new_settings, default_ranking_r assert response.typo_tolerance.enabled is False assert response.pagination == new_settings.pagination assert response.proximity_precision == new_settings.proximity_precision - # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585 - # assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" - # assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" - # assert response.embedders["test4"].source == "rest" response = await index.reset_settings() update = await async_wait_for_task(index.http_client, response.task_uid) assert update.status == "succeeded" @@ -694,10 +682,13 @@ async def test_update_embedders(async_empty_index): embedders = Embedders( embedders={ "default": UserProvidedEmbedder(dimensions=512), - # "test1": HuggingFaceEmbedder(), "test2": OpenAiEmbedder(), - # "test3": OllamaEmbedder(model="nomic-embed-text"), - "test4": RestEmbedder(url="https://myurl.com", dimensions=512), + "test4": RestEmbedder( + url="https://myurl.com", + dimensions=512, + request={"request": {"model": "minillm", "prompt": "{{text}}"}}, + response={"response": {"embedding": "{{embedding}}"}}, + ), } ) index = await async_empty_index() @@ -705,9 +696,7 @@ async def test_update_embedders(async_empty_index): await async_wait_for_task(index.http_client, response.task_uid) response = await index.get_embedders() assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" assert response.embedders["test4"].source == "rest" @@ -715,10 +704,13 @@ async def test_reset_embedders(async_empty_index): embedders = Embedders( embedders={ "default": UserProvidedEmbedder(dimensions=512), - # "test1": HuggingFaceEmbedder(), "test2": OpenAiEmbedder(), - # "test3": OllamaEmbedder(model="nomic-embed-text"), - "test4": RestEmbedder(url="https://myurl.com", dimensions=512), + "test4": RestEmbedder( + url="https://myurl.com", + dimensions=512, + request={"request": {"model": "minillm", "prompt": "{{text}}"}}, + response={"response": {"embedding": "{{embedding}}"}}, + ), } ) index = await async_empty_index() @@ -727,9 +719,7 @@ async def test_reset_embedders(async_empty_index): assert update.status == "succeeded" response = await index.get_embedders() assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" assert response.embedders["test4"].source == "rest" response = await index.reset_embedders() await async_wait_for_task(index.http_client, response.task_uid) diff --git a/tests/test_index.py b/tests/test_index.py index e57d9fe2..512ee40a 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -157,12 +157,6 @@ def test_update_settings(compress, empty_index, new_settings): assert response.non_separator_tokens == new_settings.non_separator_tokens assert response.search_cutoff_ms == new_settings.search_cutoff_ms assert response.dictionary == new_settings.dictionary - # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585 - # assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" - # assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" - # assert response.embedders["test4"].source == "rest" def test_reset_settings(empty_index, new_settings, default_ranking_rules): @@ -181,12 +175,6 @@ def test_reset_settings(empty_index, new_settings, default_ranking_rules): assert response.typo_tolerance.enabled is False assert response.pagination == new_settings.pagination assert response.proximity_precision == new_settings.proximity_precision - # TODO: Add back after embedder setting issue fixed https://github.com/meilisearch/meilisearch/issues/4585 - # assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" - # assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" - # assert response.embedders["test4"].source == "rest" response = index.reset_settings() update = wait_for_task(index.http_client, response.task_uid) assert update.status == "succeeded" @@ -679,10 +667,13 @@ def test_update_embedders(empty_index): embedders = Embedders( embedders={ "default": UserProvidedEmbedder(dimensions=512), - # "test1": HuggingFaceEmbedder(), "test2": OpenAiEmbedder(), - # "test3": OllamaEmbedder(model="nomic-embed-text"), - "test4": RestEmbedder(url="https://myurl.com", dimensions=512), + "test4": RestEmbedder( + url="https://myurl.com", + dimensions=512, + request={"request": {"model": "minillm", "prompt": "{{text}}"}}, + response={"response": {"embedding": "{{embedding}}"}}, + ), } ) index = empty_index() @@ -690,9 +681,7 @@ def test_update_embedders(empty_index): wait_for_task(index.http_client, response.task_uid) response = index.get_embedders() assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" assert response.embedders["test4"].source == "rest" @@ -700,10 +689,13 @@ def test_reset_embedders(empty_index): embedders = Embedders( embedders={ "default": UserProvidedEmbedder(dimensions=512), - # "test1": HuggingFaceEmbedder(), "test2": OpenAiEmbedder(), - # "test3": OllamaEmbedder(model="some_model"), - "test4": RestEmbedder(url="https://myurl.com", dimensions=512), + "test4": RestEmbedder( + url="https://myurl.com", + dimensions=512, + request={"request": {"model": "minillm", "prompt": "{{text}}"}}, + response={"response": {"embedding": "{{embedding}}"}}, + ), } ) index = empty_index() @@ -712,9 +704,7 @@ def test_reset_embedders(empty_index): assert update.status == "succeeded" response = index.get_embedders() assert response.embedders["default"].source == "userProvided" - # assert response.embedders["test1"].source == "huggingFace" assert response.embedders["test2"].source == "openAi" - # assert response.embedders["test3"].source == "ollama" assert response.embedders["test4"].source == "rest" response = index.reset_embedders() wait_for_task(index.http_client, response.task_uid)