Add test for optional input in Python backend

qa/L0_backend_python/python_test.py

@@ -61,6 +61,58 @@ def _infer_help(self, model_name, shape, data_type):
             output0 = result.as_numpy('OUTPUT0')
             self.assertTrue(np.all(input_data_0 == output0))
 
+    def _optional_input_infer(self, model_name, has_input0, has_input1):
+        with httpclient.InferenceServerClient("localhost:8000") as client:
+            shape = (1,)
+            if has_input0:
+                input0_numpy = np.random.randint(0,
+                                                 100,
+                                                 size=shape,
+                                                 dtype=np.int32)
+            else:
+                # Set the input0 to a default value if it is optional. This is
+                # the input used by the model if it is not provided.
+                input0_numpy = np.array([5], dtype=np.int32)
+
+            if has_input1:
+                input1_numpy = np.random.randint(0,
+                                                 100,
+                                                 size=shape,
+                                                 dtype=np.int32)
+            else:
+                # Set the input1 to a default value if it is optional. This is
+                # the input used by the model if it is not provided.
+                input1_numpy = np.array([5], dtype=np.int32)
+
+            inputs = []
+            if has_input0:
+                inputs.append(
+                    httpclient.InferInput(
+                        "INPUT0", shape,
+                        np_to_triton_dtype(input0_numpy.dtype)))
+                inputs[-1].set_data_from_numpy(input0_numpy)
+
+            if has_input1:
+                inputs.append(
+                    httpclient.InferInput(
+                        "INPUT1", shape,
+                        np_to_triton_dtype(input1_numpy.dtype)))
+                inputs[-1].set_data_from_numpy(input1_numpy)
+
+            result = client.infer(model_name, inputs)
+            output0 = result.as_numpy('OUTPUT0')
+            self.assertIsNotNone(output0, "OUTPUT0 was not found.")
+
+            output1 = result.as_numpy('OUTPUT1')
+            self.assertIsNotNone(output1, "OUTPUT1 was not found.")
+
+            expected_output0 = input0_numpy + input1_numpy
+            expected_output1 = input0_numpy - input1_numpy
+            np.testing.assert_equal(output0, expected_output0,
+                                    "OUTPUT0 doesn't match expected OUTPUT0")
+            np.testing.assert_equal(output1, expected_output1,
+                                    "OUTPUT1 doesn't match expected OUTPUT1")
+
     # We do not use a docker on Jetson so it does not impose a shared memory
     # allocation limit of 1GB. This means test will pass without the expected
     # error on jetson and is hence unnecessary.
@@ -263,6 +315,15 @@ def test_unicode(self):
                     self.assertIsNotNone(output0)
                     self.assertEqual(output0[0], input_data)
 
+    def test_optional_input(self):
+        model_name = "optional"
+
+        with self._shm_leak_detector.Probe() as shm_probe:
+            for has_input0 in [True, False]:
+                for has_input1 in [True, False]:
+                    self._optional_input_infer(model_name, has_input0,
+                                               has_input1)
+
     def test_string(self):
         model_name = "string_fixed"
         shape = [1]
@@ -290,8 +351,8 @@ def test_string(self):
 
     def test_non_contiguous(self):
         model_name = 'non_contiguous'
-        shape = [2, 64, 84, 32, 55]
-        new_shape = [64, 2, 32, 55, 84]
+        shape = [2, 10, 11, 6, 5]
+        new_shape = [10, 2, 6, 5, 11]
         shape_reorder = [1, 0, 4, 2, 3]
         with httpclient.InferenceServerClient("localhost:8000") as client:
             input_numpy = np.random.rand(*shape)

qa/L0_backend_python/test.sh

@@ -46,12 +46,14 @@ export TEST_JETSON=${TEST_JETSON:=0}
 export CUDA_VISIBLE_DEVICES=0
 
 BASE_SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1"
-SERVER_ARGS=$BASE_SERVER_ARGS
+# Set the default byte size to 5MBs to avoid going out of shared memory. The
+# environment that this job runs on has only 1GB of shared-memory available.
+SERVER_ARGS="$BASE_SERVER_ARGS --backend-config=python,shm-default-byte-size=5242880"
 
 PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG
 CLIENT_PY=./python_test.py
 CLIENT_LOG="./client.log"
-EXPECTED_NUM_TESTS="8"
+EXPECTED_NUM_TESTS="9"
 TEST_RESULT_FILE='test_results.txt'
 SERVER_LOG="./inference_server.log"
 source ../common/util.sh
@@ -108,6 +110,10 @@ mkdir -p models/init_args/1/
 cp ../python_models/init_args/model.py ./models/init_args/1/
 cp ../python_models/init_args/config.pbtxt ./models/init_args/
 
+mkdir -p models/optional/1/
+cp ../python_models/optional/model.py ./models/optional/1/
+cp ../python_models/optional/config.pbtxt ./models/optional/
+
 mkdir -p models/non_contiguous/1/
 cp ../python_models/non_contiguous/model.py ./models/non_contiguous/1/
 cp ../python_models/non_contiguous/config.pbtxt ./models/non_contiguous/config.pbtxt

qa/python_models/non_contiguous/model.py

@@ -32,7 +32,7 @@ class TritonPythonModel:
 
     def execute(self, requests):
         responses = []
-        new_shape = [64, 2, 32, 55, 84]
+        new_shape = [10, 2, 6, 5, 11]
         shape_reorder = [1, 0, 4, 2, 3]
         for request in requests:
             input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")

qa/python_models/optional/config.pbtxt

@@ -0,0 +1,55 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+name: "optional"
+backend: "python"
+max_batch_size: 0
+input [
+  {
+    name: "INPUT0"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  },
+  {
+    name: "INPUT1"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+    optional: true
+  }
+]
+output [
+  {
+    name: "OUTPUT0"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+  },
+  {
+    name: "OUTPUT1"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+  }
+]

qa/python_models/optional/model.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import triton_python_backend_utils as pb_utils
+import numpy as np
+
+
+class TritonPythonModel:
+
+    def execute(self, requests):
+        """Model supporting optional inputs. If the input is not provided, an
+        input tensor of size 1 containing scalar 5 will be used."""
+        responses = []
+        for request in requests:
+            input0_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
+            input1_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT1")
+
+            if input0_tensor is not None:
+                input0_numpy = input0_tensor.as_numpy()
+            else:
+                input0_numpy = np.array([5], dtype=np.int32)
+
+            if input1_tensor is not None:
+                input1_numpy = input1_tensor.as_numpy()
+            else:
+                input1_numpy = np.array([5], dtype=np.int32)
+
+            output0_tensor = pb_utils.Tensor("OUTPUT0",
+                                             input0_numpy + input1_numpy)
+            output1_tensor = pb_utils.Tensor("OUTPUT1",
+                                             input0_numpy - input1_numpy)
+            responses.append(
+                pb_utils.InferenceResponse([output0_tensor, output1_tensor]))
+
+        return responses