adds support for int8 and uint8 dtypes in vectorizers

docs/user_guide/04_vectorizers.ipynb

@@ -223,12 +223,12 @@
       },
       {
          "cell_type": "code",
-         "execution_count": 7,
+         "execution_count": null,
          "metadata": {},
          "outputs": [],
          "source": [
             "# You can also create many embeddings at once\n",
-            "embeddings = hf.embed_many(sentences, as_buffer=True, dtype=\"float32\")\n"
+            "embeddings = hf.embed_many(sentences, as_buffer=True)\n"
          ]
       },
       {
@@ -660,13 +660,13 @@
          "metadata": {},
          "source": [
             "## Selecting your float data type\n",
-            "When embedding text as byte arrays RedisVL supports 4 different floating point data types, `float16`, `float32`, `float64` and `bfloat16`.\n",
+            "When embedding text as byte arrays RedisVL supports 4 different floating point data types, `float16`, `float32`, `float64` and `bfloat16`, and 2 integer types, `int8` and `uint8`.\n",
             "Your dtype set for your vectorizer must match what is defined in your search index. If one is not explicitly set the default is `float32`."
          ]
       },
       {
          "cell_type": "code",
-         "execution_count": 19,
+         "execution_count": null,
          "metadata": {},
          "outputs": [],
          "source": [
@@ -675,8 +675,9 @@
             "# subsequent calls to embed('', as_buffer=True) and embed_many('', as_buffer=True) will now encode as float16\n",
             "float16_bytes = vectorizer.embed('test sentence', as_buffer=True)\n",
             "\n",
-            "# you can override this setting on each individual method call\n",
-            "float64_bytes = vectorizer.embed('test sentence', as_buffer=True, dtype=\"float64\")\n",
+            "# to generate embeddings with different dtype instantiate a new vectorizer\n",
+            "vectorizer_64 = HFTextVectorizer(dtype='float64')\n",
+            "float64_bytes = vectorizer_64.embed('test sentence', as_buffer=True)\n",
             "\n",
             "float16_bytes != float64_bytes"
          ]
@@ -690,38 +691,6 @@
             "# cleanup\n",
             "index.delete()"
          ]
-      },
-      {
-         "cell_type": "code",
-         "execution_count": null,
-         "metadata": {},
-         "outputs": [],
-         "source": []
-      },
-      {
-         "cell_type": "code",
-         "execution_count": 1,
-         "metadata": {},
-         "outputs": [],
-         "source": [
-            "dist = max(i for i in range(10))"
-         ]
-      },
-      {
-         "cell_type": "code",
-         "execution_count": null,
-         "metadata": {},
-         "outputs": [],
-         "source": [
-            "dist"
-         ]
-      },
-      {
-         "cell_type": "code",
-         "execution_count": null,
-         "metadata": {},
-         "outputs": [],
-         "source": []
       }
    ],
    "metadata": {
@@ -746,4 +715,4 @@
    },
    "nbformat": 4,
    "nbformat_minor": 2
-}
+}

redisvl/redis/utils.py

@@ -41,7 +41,7 @@ def array_to_buffer(array: List[float], dtype: str) -> bytes:
         raise ValueError(
             f"Invalid data type: {dtype}. Supported types are: {[t.lower() for t in VectorDataType]}"
         )
-    return np.array(array).astype(dtype.lower()).tobytes()
+    return np.array(array, dtype=dtype.lower()).tobytes()
 
 
 def buffer_to_array(buffer: bytes, dtype: str) -> List[Any]:

redisvl/schema/fields.py

@@ -30,6 +30,8 @@ class VectorDataType(str, Enum):
     FLOAT16 = "FLOAT16"
     FLOAT32 = "FLOAT32"
     FLOAT64 = "FLOAT64"
+    INT8 = "INT8"
+    UINT8 = "UINT8"
 
 
 class VectorIndexAlgorithm(str, Enum):

redisvl/utils/vectorize/text/azureopenai.py

@@ -5,6 +5,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 # ignore that openai isn't imported
@@ -167,6 +168,7 @@ def _set_model_dims(self, model) -> int:
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],
@@ -213,6 +215,7 @@ def embed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -251,6 +254,7 @@ def embed(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed_many(
         self,
         texts: List[str],
@@ -299,6 +303,7 @@ async def aembed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed(
         self,
         text: str,

redisvl/utils/vectorize/text/bedrock.py

@@ -6,6 +6,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 
@@ -118,6 +119,7 @@ def _set_model_dims(self, model: str) -> int:
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -158,6 +160,7 @@ def embed(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],

redisvl/utils/vectorize/text/cohere.py

@@ -5,6 +5,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 # ignore that cohere isn't imported
@@ -111,6 +112,7 @@ def _set_model_dims(self, model) -> int:
             raise ValueError(f"Error setting embedding model dimensions: {str(e)}")
         return len(embedding)
 
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -177,6 +179,7 @@ def embed(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],

redisvl/utils/vectorize/text/custom.py

@@ -2,6 +2,7 @@
 
 from pydantic.v1 import PrivateAttr
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 
@@ -155,6 +156,7 @@ def _validate_sync_callables(self) -> int:
 
         return dims
 
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -191,6 +193,7 @@ def embed(
 
         return self._process_embedding(result, as_buffer, dtype)
 
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],
@@ -239,6 +242,7 @@ def embed_many(
         return embeddings
 
     @validate_async
+    @deprecated_argument("dtype")
     async def aembed(
         self,
         text: str,
@@ -280,6 +284,7 @@ async def aembed(
         return self._process_embedding(result, as_buffer, dtype)
 
     @validate_async
+    @deprecated_argument("dtype")
     async def aembed_many(
         self,
         texts: List[str],

redisvl/utils/vectorize/text/huggingface.py

@@ -2,6 +2,7 @@
 
 from pydantic.v1 import PrivateAttr
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 
@@ -79,6 +80,7 @@ def _set_model_dims(self):
             raise ValueError(f"Error setting embedding model dimensions: {str(e)}")
         return len(embedding)
 
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -112,6 +114,7 @@ def embed(
         embedding = self._client.encode([text], **kwargs)[0]
         return self._process_embedding(embedding.tolist(), as_buffer, dtype)
 
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],

redisvl/utils/vectorize/text/mistral.py

@@ -5,6 +5,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 # ignore that mistralai isn't imported
@@ -116,6 +117,7 @@ def _set_model_dims(self, model) -> int:
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],
@@ -162,6 +164,7 @@ def embed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -200,6 +203,7 @@ def embed(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed_many(
         self,
         texts: List[str],
@@ -248,6 +252,7 @@ async def aembed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed(
         self,
         text: str,

redisvl/utils/vectorize/text/openai.py

@@ -5,6 +5,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 # ignore that openai isn't imported
@@ -121,6 +122,7 @@ def _set_model_dims(self, model) -> int:
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],
@@ -167,6 +169,7 @@ def embed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,
@@ -205,6 +208,7 @@ def embed(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed_many(
         self,
         texts: List[str],
@@ -253,6 +257,7 @@ async def aembed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     async def aembed(
         self,
         text: str,

redisvl/utils/vectorize/text/vertexai.py

@@ -5,6 +5,7 @@
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 from tenacity.retry import retry_if_not_exception_type
 
+from redisvl.utils.utils import deprecated_argument
 from redisvl.utils.vectorize.base import BaseVectorizer
 
 
@@ -128,6 +129,7 @@ def _set_model_dims(self) -> int:
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed_many(
         self,
         texts: List[str],
@@ -173,6 +175,7 @@ def embed_many(
         stop=stop_after_attempt(6),
         retry=retry_if_not_exception_type(TypeError),
     )
+    @deprecated_argument("dtype")
     def embed(
         self,
         text: str,