add parallelizable UDF decorators

eva/udfs/abstract/tracker_abstract_udf.py

@@ -31,7 +31,7 @@ class EVATrackerAbstractUDF(AbstractUDF):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    @setup(cacheable=False, udf_type="object_tracker", batchable=False)
+    @setup(cacheable=False, udf_type="object_tracker", batchable=False, parallelizable=False)
     def setup(self, *args, **kwargs):
         super().setup(*args, **kwargs)
 

eva/udfs/chatgpt.py

@@ -38,7 +38,7 @@ class ChatGPT(AbstractUDF):
     def name(self) -> str:
         return "ChatGPT"
 
-    @setup(cacheable=False, udf_type="chat-completion", batchable=True)
+    @setup(cacheable=False, udf_type="chat-completion", batchable=True, parallelizable=False)
     def setup(
         self,
         model="gpt-3.5-turbo",

eva/udfs/decorators/decorators.py

@@ -19,14 +19,15 @@
 from eva.udfs.decorators.io_descriptors.abstract_types import IOArgument
 
 
-def setup(cacheable: bool = False, udf_type: str = "Abstract", batchable: bool = True):
+def setup(cacheable: bool = False, udf_type: str = "Abstract", batchable: bool = True, parallelizable: bool = True):
     """decorator for the setup function. It will be used to set the cache, batching and
     udf_type parameters in the catalog
 
     Args:
-        use_cache (bool): True if the udf should be cached
+        cacheable (bool): True if the udf should be cached
         udf_type (str): Type of the udf
-        batch (bool): True if the udf should be batched
+        batchable (bool): True if the udf should be batched
+        parallelizable (bool): True if the udf can be parallelized during the evaluation
     """
 
     def inner_fn(arg_fn):
@@ -38,6 +39,7 @@ def wrapper(*args, **kwargs):
         tags["cacheable"] = cacheable
         tags["udf_type"] = udf_type
         tags["batchable"] = batchable
+        tags["parallelizable"] = parallelizable
         wrapper.tags = tags
         return wrapper
 

eva/udfs/ndarray/annotate.py

@@ -26,7 +26,7 @@
 
 
 class Annotate(AbstractUDF):
-    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True)
+    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True, parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/array_count.py

@@ -16,13 +16,14 @@
 import pandas as pd
 
 from eva.udfs.abstract.abstract_udf import AbstractUDF
-
+from eva.udfs.decorators.decorators import forward, setup
 
 class ArrayCount(AbstractUDF):
     @property
     def name(self) -> str:
         return "ArrayCount"
 
+    @setup(parallelizable=False)
     def setup(self):
         pass
 

eva/udfs/ndarray/crop.py

@@ -16,9 +16,11 @@
 import pandas as pd
 
 from eva.udfs.abstract.abstract_udf import AbstractUDF
+from eva.udfs.decorators.decorators import forward, setup
 
 
 class Crop(AbstractUDF):
+    @setup(parallelizable=False)
     def setup(self):
         pass
 

eva/udfs/ndarray/fuzzy_join.py

@@ -19,6 +19,7 @@
 
 
 class FuzzDistance(AbstractUDF):
+    @setup(parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/gaussian_blur.py

@@ -23,7 +23,7 @@
 
 
 class GaussianBlur(AbstractUDF):
-    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True)
+    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True, parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/horizontal_flip.py

@@ -23,7 +23,7 @@
 
 
 class HorizontalFlip(AbstractUDF):
-    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True)
+    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True, parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/open.py

@@ -17,9 +17,11 @@
 import pandas as pd
 
 from eva.udfs.abstract.abstract_udf import AbstractUDF
+from eva.udfs.decorators.decorators import forward, setup
 
 
 class Open(AbstractUDF):
+    @setup(parallelizable=True)
     def setup(self):
         # cache data to avoid expensive open files on disk
         self._data_cache = dict()

eva/udfs/ndarray/similarity.py

@@ -22,6 +22,7 @@ class Similarity(AbstractUDF):
     def _get_distance(self, numpy_distance):
         return numpy_distance[0][0]
 
+    @setup(parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/to_grayscale.py

@@ -23,7 +23,7 @@
 
 
 class ToGrayscale(AbstractUDF):
-    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True)
+    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True, parallelizable=True)
     def setup(self):
         pass
 

eva/udfs/ndarray/vertical_flip.py

@@ -23,7 +23,7 @@
 
 
 class VerticalFlip(AbstractUDF):
-    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True)
+    @setup(cacheable=False, udf_type="cv2-transformation", batchable=True, parallelizable=True)
     def setup(self):
         pass
 

test/udfs/decorators/test_decorators.py

@@ -21,14 +21,15 @@
 
 class DecoratorTests(unittest.TestCase):
     def test_setup_flags_are_updated(self):
-        @setup(cacheable=True, udf_type="classification", batchable=True)
+        @setup(cacheable=True, udf_type="classification", batchable=True, parallelizable=False)
         def setup_func():
             pass
 
         setup_func()
         self.assertTrue(setup_func.tags["cacheable"])
         self.assertTrue(setup_func.tags["batchable"])
         self.assertEqual(setup_func.tags["udf_type"], "classification")
+        self.assertEqual(setup_func.tags["parallelizable"], False)
 
     def test_setup_flags_are_updated_with_default_values(self):
         @setup()
@@ -39,6 +40,7 @@ def setup_func():
         self.assertFalse(setup_func.tags["cacheable"])
         self.assertTrue(setup_func.tags["batchable"])
         self.assertEqual(setup_func.tags["udf_type"], "Abstract")
+        self.assertEqual(setup_func.tags["parallelizable"], True)
 
     def test_forward_flags_are_updated(self):
         input_type = PandasDataframe(