fix(aten::instance_norm): Handle optional inputs in instance norm con…

.github/scripts/generate_binary_build_matrix.py

@@ -469,7 +469,6 @@ def generate_wheels_matrix(
     ret: List[Dict[str, Any]] = []
     for python_version in python_versions:
         for arch_version in arches:
-
             # TODO: Enable Python 3.13 support for ROCM
             if arch_version in ROCM_ARCHES and python_version == "3.13":
                 continue

core/conversion/converters/impl/batch_norm.cpp

@@ -134,9 +134,14 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
 
               auto eps = static_cast<float>(args[7].unwrapToDouble(1e-5f));
 
-              auto scales = args[1].unwrapToTensor(at::ones(shape[1], options)).cpu().contiguous();
-              auto bias = args[2].unwrapToTensor(at::zeros(shape[1], options)).cpu().contiguous();
-
+              auto scales = at::ones(shape[1], options);
+              if (!args[1].IValue()->isNone()) {
+                scales = args[1].unwrapToTensor(at::ones(shape[1], options)).cpu().contiguous();
+              }
+              auto bias = at::zeros(shape[1], options);
+              if (!args[2].IValue()->isNone()) {
+                bias = args[2].unwrapToTensor(at::zeros(shape[1], options)).cpu().contiguous();
+              }
               // track_running_stats=True
               if (!args[3].IValue()->isNone() || !args[4].IValue()->isNone()) {
                 auto running_mean = args[3].unwrapToTensor();
@@ -154,6 +159,8 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
                 return true;
               }
 
+              // Not sure this actually does something since the cudnn_enabled is from the PyTorch context.
+              // We need cuDNN either way to run this converter
               auto cudnn_enabled = static_cast<bool>(args[8].unwrapToBool(false));
               if (!cudnn_enabled) {
                 LOG_DEBUG(
@@ -162,7 +169,7 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
                     so for some functionalities, users need to install correct \
                     cuDNN version by themselves. Please see our support matrix \
                     here: https://docs.nvidia.com/deeplearning/tensorrt/support-matrix/index.html.");
-                return false;
+                // return false;
               }
 
               const int relu = 0;

core/util/prelude.h

@@ -2,6 +2,7 @@
 
 // A collection of headers from util that will typically get included in most
 // files
+#include <cstdint>
 #include "core/util/Exception.h"
 #include "core/util/build_info.h"
 #include "core/util/jit_util.h"

docs/_downloads/0e30a6276601af7e5fc4d5166e2e3d37/torch_compile_advanced_usage.py

@@ -4,7 +4,8 @@
 Torch Compile Advanced Usage
 ======================================================
 
-This interactive script is intended as an overview of the process by which `torch_tensorrt.compile(..., ir="torch_compile", ...)` works, and how it integrates with the `torch.compile` API."""
+This interactive script is intended as an overview of the process by which `torch_tensorrt.compile(..., ir="torch_compile", ...)` works, and how it integrates with the `torch.compile` API.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/2a9ac10f2667047a7f398d1593b7ca33/torch_export_gpt2.py

@@ -4,7 +4,8 @@
 Compiling GPT2 using the dynamo backend
 ==========================================================
 
-This script illustrates Torch-TensorRT workflow with dynamo backend on popular GPT2 model."""
+This script illustrates Torch-TensorRT workflow with dynamo backend on popular GPT2 model.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/3d4d74f6636d986f33167154f6553961/torch_export_cudagraphs.py

@@ -4,7 +4,8 @@
 Torch Export with Cudagraphs
 ======================================================
 
-This interactive script is intended as an overview of the process by which the Torch-TensorRT Cudagraphs integration can be used in the `ir="dynamo"` path. The functionality works similarly in the `torch.compile` path as well."""
+This interactive script is intended as an overview of the process by which the Torch-TensorRT Cudagraphs integration can be used in the `ir="dynamo"` path. The functionality works similarly in the `torch.compile` path as well.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/418941399c146271a7b7728ba3059960/dynamo_compile_resnet_example.py

@@ -4,7 +4,8 @@
 Compiling ResNet using the Torch-TensorRT Dyanmo Frontend
 ==========================================================
 
-This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a ResNet model."""
+This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a ResNet model.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/7b7004dc2ea6f839be532665e16e0426/torch_export_llama2.py

@@ -4,7 +4,8 @@
 Compiling Llama2 using the dynamo backend
 ==========================================================
 
-This script illustrates Torch-TensorRT workflow with dynamo backend on popular Llama2 model."""
+This script illustrates Torch-TensorRT workflow with dynamo backend on popular Llama2 model.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/c0341280f3b022df00c4241c42d9ee8b/custom_kernel_plugins.py

@@ -316,7 +316,6 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
 
 import cupy as cp  # Needed to work around API gaps in PyTorch to build torch.Tensors around preallocated CUDA memory
 import numpy as np
-
 import tensorrt as trt
 
 
@@ -348,7 +347,6 @@ def get_output_dimensions(
         inputs: List[trt.DimsExprs],
         exprBuilder: trt.IExprBuilder,
     ) -> trt.DimsExprs:
-
         output_dims = trt.DimsExprs(inputs[0])
 
         for i in range(np.size(self.pads) // 2):
@@ -404,7 +402,6 @@ def enqueue(
         workspace: int,
         stream: int,
     ) -> None:
-
         # Host code is slightly different as this will be run as part of the TRT execution
         in_dtype = torchtrt.dtype.try_from(input_desc[0].type).to(np.dtype)
 
@@ -528,7 +525,6 @@ def circular_padding_converter(
     kwargs: Dict[str, Argument],
     name: str,
 ):
-
     # How to retrieve a plugin if it is defined elsewhere (e.g. linked library)
     plugin_registry = trt.get_plugin_registry()
     plugin_creator = plugin_registry.get_plugin_creator(

docs/_downloads/d6e1bb6ec5f884994554d9d12e37a0f6/torch_compile_resnet_example.py

@@ -4,7 +4,8 @@
 Compiling ResNet with dynamic shapes using the `torch.compile` backend
 ==========================================================
 
-This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a ResNet model."""
+This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a ResNet model.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/dfa60e8f9850fd7761f3e7da81304d32/torch_compile_transformers_example.py

@@ -4,7 +4,8 @@
 Compiling BERT using the `torch.compile` backend
 ==============================================================
 
-This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a BERT model."""
+This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a BERT model.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/e1ef5a42560a98a132f56a79d0b66f79/dynamo_compile_advanced_usage.py

@@ -4,7 +4,8 @@
 Dynamo Compile Advanced Usage
 ======================================================
 
-This interactive script is intended as an overview of the process by which `torch_tensorrt.dynamo.compile` works, and how it integrates with the new `torch.compile` API."""
+This interactive script is intended as an overview of the process by which `torch_tensorrt.dynamo.compile` works, and how it integrates with the new `torch.compile` API.
+"""
 
 # %%
 # Imports and Model Definition

docs/_downloads/e550c5f53cc43e11aa6da8cfb79b54df/dynamo_compile_transformers_example.py

@@ -4,7 +4,8 @@
 Compiling a Transformer using torch.compile and TensorRT
 ==============================================================
 
-This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a transformer-based model."""
+This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a transformer-based model.
+"""
 
 # %%
 # Imports and Model Definition

docs/v1.4.0/_downloads/418941399c146271a7b7728ba3059960/dynamo_compile_resnet_example.py

@@ -4,7 +4,8 @@
 Compiling ResNet using the Torch-TensorRT Dyanmo Frontend
 ==========================================================
 
-This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a ResNet model."""
+This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a ResNet model.
+"""
 
 # %%
 # Imports and Model Definition

docs/v1.4.0/_downloads/e1ef5a42560a98a132f56a79d0b66f79/dynamo_compile_advanced_usage.py

@@ -4,7 +4,8 @@
 Dynamo Compile Advanced Usage
 ======================================================
 
-This interactive script is intended as an overview of the process by which `torch_tensorrt.dynamo.compile` works, and how it integrates with the new `torch.compile` API."""
+This interactive script is intended as an overview of the process by which `torch_tensorrt.dynamo.compile` works, and how it integrates with the new `torch.compile` API.
+"""
 
 # %%
 # Imports and Model Definition

docs/v1.4.0/_downloads/e550c5f53cc43e11aa6da8cfb79b54df/dynamo_compile_transformers_example.py

@@ -4,7 +4,8 @@
 Compiling a Transformer using torch.compile and TensorRT
 ==============================================================
 
-This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a transformer-based model."""
+This interactive script is intended as a sample of the `torch_tensorrt.dynamo.compile` workflow on a transformer-based model.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/custom_kernel_plugins.py

@@ -316,7 +316,6 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
 
 import cupy as cp  # Needed to work around API gaps in PyTorch to build torch.Tensors around preallocated CUDA memory
 import numpy as np
-
 import tensorrt as trt
 
 
@@ -348,7 +347,6 @@ def get_output_dimensions(
         inputs: List[trt.DimsExprs],
         exprBuilder: trt.IExprBuilder,
     ) -> trt.DimsExprs:
-
         output_dims = trt.DimsExprs(inputs[0])
 
         for i in range(np.size(self.pads) // 2):
@@ -404,7 +402,6 @@ def enqueue(
         workspace: int,
         stream: int,
     ) -> None:
-
         # Host code is slightly different as this will be run as part of the TRT execution
         in_dtype = torchtrt.dtype.try_from(input_desc[0].type).to(np.dtype)
 
@@ -528,7 +525,6 @@ def circular_padding_converter(
     kwargs: Dict[str, Argument],
     name: str,
 ):
-
     # How to retrieve a plugin if it is defined elsewhere (e.g. linked library)
     plugin_registry = trt.get_plugin_registry()
     plugin_creator = plugin_registry.get_plugin_creator(

examples/dynamo/torch_compile_advanced_usage.py

@@ -4,7 +4,8 @@
 Torch Compile Advanced Usage
 ======================================================
 
-This interactive script is intended as an overview of the process by which `torch_tensorrt.compile(..., ir="torch_compile", ...)` works, and how it integrates with the `torch.compile` API."""
+This interactive script is intended as an overview of the process by which `torch_tensorrt.compile(..., ir="torch_compile", ...)` works, and how it integrates with the `torch.compile` API.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/torch_compile_resnet_example.py

@@ -4,7 +4,8 @@
 Compiling ResNet with dynamic shapes using the `torch.compile` backend
 ==========================================================
 
-This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a ResNet model."""
+This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a ResNet model.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/torch_compile_transformers_example.py

@@ -4,7 +4,8 @@
 Compiling BERT using the `torch.compile` backend
 ==============================================================
 
-This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a BERT model."""
+This interactive script is intended as a sample of the Torch-TensorRT workflow with `torch.compile` on a BERT model.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/torch_export_cudagraphs.py

@@ -4,7 +4,8 @@
 Torch Export with Cudagraphs
 ======================================================
 
-This interactive script is intended as an overview of the process by which the Torch-TensorRT Cudagraphs integration can be used in the `ir="dynamo"` path. The functionality works similarly in the `torch.compile` path as well."""
+This interactive script is intended as an overview of the process by which the Torch-TensorRT Cudagraphs integration can be used in the `ir="dynamo"` path. The functionality works similarly in the `torch.compile` path as well.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/torch_export_gpt2.py

@@ -4,7 +4,8 @@
 Compiling GPT2 using the dynamo backend
 ==========================================================
 
-This script illustrates Torch-TensorRT workflow with dynamo backend on popular GPT2 model."""
+This script illustrates Torch-TensorRT workflow with dynamo backend on popular GPT2 model.
+"""
 
 # %%
 # Imports and Model Definition

examples/dynamo/torch_export_llama2.py

@@ -4,7 +4,8 @@
 Compiling Llama2 using the dynamo backend
 ==========================================================
 
-This script illustrates Torch-TensorRT workflow with dynamo backend on popular Llama2 model."""
+This script illustrates Torch-TensorRT workflow with dynamo backend on popular Llama2 model.
+"""
 
 # %%
 # Imports and Model Definition

notebooks/CitriNet-example.ipynb

@@ -384,12 +384,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import nemo\n",
     "import torch\n",
     "\n",
     "import nemo.collections.asr as nemo_asr\n",
     "from nemo.core import typecheck\n",
-    "typecheck.set_typecheck_enabled(False) "
+    "typecheck.set_typecheck_enabled(False)"
    ]
   },
   {
@@ -572,11 +571,8 @@
     "from __future__ import absolute_import\n",
     "from __future__ import division\n",
     "\n",
-    "import argparse\n",
     "import timeit\n",
     "import numpy as np\n",
-    "import torch\n",
-    "import torch_tensorrt as trtorch\n",
     "import torch.backends.cudnn as cudnn\n",
     "\n",
     "def benchmark(model, input_tensor, num_loops, model_name, batch_size):\n",
@@ -632,7 +628,7 @@
     "    else:\n",
     "        model_name = f\"{variant}.ts\"\n",
     "\n",
-    "    print(f\"Loading model: {model_name}\") \n",
+    "    print(f\"Loading model: {model_name}\")\n",
     "    # Load traced model to CPU first\n",
     "    model = torch.jit.load(model_name).cuda()\n",
     "    cudnn.benchmark = True\n",
@@ -727,9 +723,7 @@
    ],
    "source": [
     "import torch\n",
-    "import torch.nn as nn\n",
     "import torch_tensorrt as torchtrt\n",
-    "import argparse\n",
     "\n",
     "variant = \"stt_en_citrinet_256\"\n",
     "precisions = [torch.float, torch.half]\n",
@@ -827,7 +821,7 @@
     "    else:\n",
     "        model_name = f\"{variant}.ts\"\n",
     "\n",
-    "    print(f\"Loading model: {model_name}\") \n",
+    "    print(f\"Loading model: {model_name}\")\n",
     "    # Load traced model to CPU first\n",
     "    model = torch.jit.load(model_name).cuda()\n",
     "    cudnn.benchmark = True\n",
@@ -906,7 +900,7 @@
     "    else:\n",
     "        model_name = f\"{variant}.ts\"\n",
     "\n",
-    "    print(f\"Loading model: {model_name}\") \n",
+    "    print(f\"Loading model: {model_name}\")\n",
     "    # Load traced model to CPU first\n",
     "    model = torch.jit.load(model_name).cuda()\n",
     "    cudnn.benchmark = True\n",