Merge pull request nebuly-ai#157 from valeriosofi/main

Bugfixes

apps/accelerate/speedster/speedster/api/functions.py

@@ -9,40 +9,14 @@
     Optional,
 )
 
-from loguru import logger
-
 from nebullvm.config import DEFAULT_METRIC_DROP_THS
 from nebullvm.optional_modules.tensorflow import tensorflow as tf
 from nebullvm.optional_modules.torch import torch
-from nebullvm.tools.base import Device
 from nebullvm.tools.logger import debug_mode_enabled, LoggingContext
-from nebullvm.tools.utils import gpu_is_available
 
 from speedster.root_op import SpeedsterRootOp
 
-
-def _check_device(device: Optional[str]) -> Device:
-    if device is None:
-        if gpu_is_available():
-            device = Device.GPU
-        else:
-            device = Device.CPU
-    else:
-        if device.lower() == "gpu":
-            if not gpu_is_available():
-                logger.warning(
-                    "Selected GPU device but no available GPU found on this "
-                    "platform. CPU will be used instead. Please make sure "
-                    "that the gpu is installed and can be used by your "
-                    "framework."
-                )
-                device = Device.CPU
-            else:
-                device = Device.GPU
-        else:
-            device = Device.CPU
-
-    return device
+from nebullvm.tools.utils import check_device
 
 
 def optimize_model(
@@ -145,7 +119,7 @@ def optimize_model(
             take as input and it will return `torch.Tensor`s.
     """
     root_op = SpeedsterRootOp()
-    device = _check_device(device)
+    device = check_device(device)
 
     disable_log = True if not debug_mode_enabled() else False
 

nebullvm/installers/install_tensor_rt.sh

@@ -4,7 +4,6 @@ if [[ "$(grep '^ID_LIKE' /etc/os-release)" == *"centos"* ]]
 then
   # Installation for centos type linux distribution
   # Try installation with pip if fails then install from source
-  pip3 install --upgrade setuptools pip
   python3 -m pip install --upgrade tensorrt
   pip3 install colored polygraphy --extra-index-url https://pypi.ngc.nvidia.com
 
@@ -21,7 +20,6 @@ then
   fi
 else
   # Try installation with pip if fails then install from source
-  pip install --upgrade setuptools pip
   python3 -m pip install --upgrade tensorrt
   pip install colored polygraphy --extra-index-url https://pypi.ngc.nvidia.com
 

nebullvm/operations/inference_learners/onnx.py

@@ -171,7 +171,7 @@ def load(cls, path: Union[Path, str], **kwargs):
         onnx_path = path / ONNX_FILENAMES["model_name"]
         metadata = LearnerMetadata.read(path)
         input_tfms = metadata.input_tfms
-        device = metadata.device
+        device = Device(metadata.device)
         if input_tfms is not None:
             input_tfms = MultiStageTransformation.from_dict(
                 metadata.input_tfms

nebullvm/operations/inference_learners/pytorch.py

@@ -76,7 +76,7 @@ def load(cls, path: Union[Path, str], **kwargs):
             input_tfms=MultiStageTransformation.from_dict(metadata.input_tfms)
             if metadata.input_tfms is not None
             else None,
-            device=metadata.device,
+            device=Device(metadata.device),
         )
 
     @classmethod

nebullvm/operations/inference_learners/tensorflow.py

@@ -45,7 +45,7 @@ def load(cls, path: Union[Path, str], **kwargs):
         model = tf.keras.models.load_model(
             path / TENSORFLOW_BACKEND_FILENAMES["tf_model"]
         )
-        device = metadata.device
+        device = Device(metadata.device)
         return cls(
             tf_model=model,
             network_parameters=network_parameters,
@@ -108,7 +108,7 @@ def load(cls, path: Union[Path, str], **kwargs):
         metadata = LearnerMetadata.read(path)
         network_parameters = ModelParams(**metadata.network_parameters)
         input_tfms = metadata.input_tfms
-        device = metadata.device
+        device = Device(metadata.device)
         return cls(
             tflite_file=tflite_file,
             network_parameters=network_parameters,

nebullvm/tools/benchmark.py

@@ -6,9 +6,10 @@
 from loguru import logger
 from tqdm import tqdm
 
+from nebullvm.operations.inference_learners.base import BaseInferenceLearner
 from nebullvm.optional_modules.tensorflow import tensorflow as tf
 from nebullvm.optional_modules.torch import torch, DataLoader
-from nebullvm.tools.base import DeepLearningFramework, ModelParams
+from nebullvm.tools.base import DeepLearningFramework, ModelParams, Device
 from nebullvm.tools.data import DataManager
 from nebullvm.tools.onnx import create_model_inputs_onnx
 from nebullvm.tools.pytorch import create_model_inputs_torch
@@ -17,6 +18,7 @@
     check_input_data,
     extract_info_from_data,
     is_data_subscriptable,
+    check_device,
 )
 
 
@@ -55,9 +57,10 @@ def _create_model_inputs(
 
 
 class BaseBenchmark(ABC):
-    def __init__(self, model, input_tensors, n_warmup=50, n_runs=1000):
+    def __init__(self, model, input_tensors, device, n_warmup=50, n_runs=1000):
         self.model = model
         self.input_tensors = input_tensors
+        self.device = device
         self.n_warmup = n_warmup
         self.n_runs = n_runs
 
@@ -68,17 +71,13 @@ def benchmark(self):
 
 class PytorchBenchmark(BaseBenchmark):
     def benchmark(self):
-        has_cuda = torch.cuda.is_available()
-        device = torch.device("cuda" if has_cuda else "cpu")
+        device = torch.device("cuda" if self.device is Device.GPU else "cpu")
         input_tensors = [
             [tensor.to(device) for tensor in tensors]
             for tensors in self.input_tensors
         ]
         batch_size = input_tensors[0][0].shape[0]
 
-        if isinstance(self.model, torch.nn.Module):
-            self.model.to(device).eval()
-
         if isinstance(self.model, torch.nn.Module):
             self.model.to(device).eval()
 
@@ -90,7 +89,7 @@ def benchmark(self):
                 self.model(
                     *input_tensors[i % min(self.n_warmup, len(input_tensors))]
                 )
-        if has_cuda:
+        if self.device is Device.GPU:
             torch.cuda.synchronize()
         timings = []
         with torch.no_grad():
@@ -102,7 +101,7 @@ def benchmark(self):
                 self.model(
                     *input_tensors[i % min(self.n_runs, len(input_tensors))]
                 )
-                if has_cuda:
+                if self.device is Device.GPU:
                     torch.cuda.synchronize()
                 end_time = time.time()
                 timings.append(end_time - start_time)
@@ -159,9 +158,12 @@ def benchmark(
         n_runs (int, optional): Number of iterations performed to benchmark
             the model.
     """
-    if device is None:
-        logger.warning("No device specified, using CPU")
-        device = "cpu"
+    if not isinstance(model, BaseInferenceLearner):
+        device = check_device(device)
+    else:
+        device = model.device
+
+    logger.info(f"Running benchmark on {device.name}")
 
     dl_framework = _get_dl_framework(model)
 
@@ -213,6 +215,7 @@ def benchmark(
     BENCHMARK_FUNCTIONS[dl_framework](
         model=model,
         input_tensors=input_data,
+        device=device,
         n_warmup=n_warmup,
         n_runs=n_runs,
     ).benchmark()

nebullvm/tools/utils.py

@@ -1,9 +1,6 @@
 import subprocess
 import sys
 from pathlib import Path
-
-import numpy as np
-from packaging import version
 from types import ModuleType
 from typing import (
     Tuple,
@@ -17,6 +14,10 @@
     Callable,
 )
 
+import numpy as np
+from loguru import logger
+from packaging import version
+
 from nebullvm.optional_modules.tensorflow import tensorflow as tf
 from nebullvm.optional_modules.torch import torch
 from nebullvm.tools.base import DeepLearningFramework, Device, ModelParams
@@ -185,6 +186,30 @@ def is_dict_type(data_sample: Any):
         return True
 
 
+def check_device(device: Optional[str]) -> Device:
+    if device is None:
+        if gpu_is_available():
+            device = Device.GPU
+        else:
+            device = Device.CPU
+    else:
+        if device.lower() == "gpu":
+            if not gpu_is_available():
+                logger.warning(
+                    "Selected GPU device but no available GPU found on this "
+                    "platform. CPU will be used instead. Please make sure "
+                    "that the gpu is installed and can be used by your "
+                    "framework."
+                )
+                device = Device.CPU
+            else:
+                device = Device.GPU
+        else:
+            device = Device.CPU
+
+    return device
+
+
 INFO_EXTRACTION_DICT: Dict[DeepLearningFramework, Callable] = {
     DeepLearningFramework.PYTORCH: extract_info_from_torch_data,
     DeepLearningFramework.TENSORFLOW: extract_info_from_tf_data,

notebooks/speedster/README.md

@@ -67,7 +67,7 @@ docker pull nebulydocker/nebullvm:latest
 ```
 Once pulled, the container can be launched with the following command:
 ```
-docker run --rm --gpus all -ti -v $PWD:/nebullvm nebulydocker/nebullvm:latest
+docker run --rm --gpus all -ti -p 8888:8888 -v $PWD:/nebullvm nebulydocker/nebullvm:latest
 ```
 The `-v` option in the command above allows to persist all the changes that will be done to the notebooks inside the container.
 Please note that, in order to enable gpu inside docker, you have to ensure that nvidia docker is installed. Please follow the "Setting up NVIDIA Container Toolkit" part from the 

notebooks/speedster/huggingface/Accelerate_Hugging_Face_PyTorch_BERT_with_Speedster.ipynb

@@ -258,7 +258,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         ")"
       ]
@@ -447,7 +447,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         "    metric_drop_ths=0.1,\n",
         ")"
@@ -612,6 +612,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "id": "b77ff2ac",
       "metadata": {
@@ -624,9 +625,9 @@
         "</center>\n",
         "\n",
         "<center> \n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#how-it-works\" target=\"_blank\" style=\"text-decoration: none;\"> How nebullvm works </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#api-quick-view\" target=\"_blank\" style=\"text-decoration: none;\"> API quick view </a> \n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#key-concepts\" target=\"_blank\" style=\"text-decoration: none;\"> How speedster works </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#quick-start\" target=\"_blank\" style=\"text-decoration: none;\"> Quick start </a> \n",
         "</center>"
       ]
     }

notebooks/speedster/huggingface/Accelerate_Hugging_Face_PyTorch_DistilBERT_with_Speedster.ipynb

@@ -258,7 +258,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         ")"
       ]
@@ -447,7 +447,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         "    metric_drop_ths=0.1,\n",
         ")"
@@ -612,6 +612,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "id": "b77ff2ac",
       "metadata": {
@@ -624,9 +625,9 @@
         "</center>\n",
         "\n",
         "<center> \n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#how-it-works\" target=\"_blank\" style=\"text-decoration: none;\"> How nebullvm works </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#api-quick-view\" target=\"_blank\" style=\"text-decoration: none;\"> API quick view </a> \n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#key-concepts\" target=\"_blank\" style=\"text-decoration: none;\"> How speedster works </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#quick-start\" target=\"_blank\" style=\"text-decoration: none;\"> Quick start </a> \n",
         "</center>"
       ]
     }
@@ -653,7 +654,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.8.9"
+      "version": "3.8.9 (default, Apr 13 2022, 08:48:06) \n[Clang 13.1.6 (clang-1316.0.21.2.5)]"
     },
     "vscode": {
       "interpreter": {

notebooks/speedster/huggingface/Accelerate_Hugging_Face_PyTorch_GPT2_with_Speedster.ipynb

@@ -267,7 +267,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         ")"
       ]
@@ -436,7 +436,7 @@
         "    model=model,\n",
         "    input_data=encoded_inputs,\n",
         "    optimization_time=\"constrained\",\n",
-        "    ignore_compilers=[\"tensor RT\"],  # TensorRT does not work for this model\n",
+        "    ignore_compilers=[\"tensor_rt\", \"tvm\"],  # TensorRT does not work for this model\n",
         "    dynamic_info=dynamic_info,\n",
         "    metric_drop_ths=0.1,\n",
         ")"
@@ -585,6 +585,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "id": "b77ff2ac",
       "metadata": {
@@ -597,9 +598,9 @@
         "</center>\n",
         "\n",
         "<center> \n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#how-it-works\" target=\"_blank\" style=\"text-decoration: none;\"> How nebullvm works </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
-        "    <a href=\"https://github.com/nebuly-ai/nebullvm#api-quick-view\" target=\"_blank\" style=\"text-decoration: none;\"> API quick view </a> \n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#key-concepts\" target=\"_blank\" style=\"text-decoration: none;\"> How speedster works </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#documentation\" target=\"_blank\" style=\"text-decoration: none;\"> Documentation </a> •\n",
+        "    <a href=\"https://github.com/nebuly-ai/nebullvm/tree/main/apps/accelerate/speedster#quick-start\" target=\"_blank\" style=\"text-decoration: none;\"> Quick start </a> \n",
         "</center>"
       ]
     }