triton-inference-server · Tabrizian · Jun 2, 2021 · May 31, 2021
diff --git a/qa/L0_backend_python/common.sh b/qa/L0_backend_python/common.sh
@@ -0,0 +1,58 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+get_shm_pages() {
+  shm_pages=(`ls /dev/shm`)
+  echo ${#shm_pages[@]}
+}
+
+install_conda() {
+  rm -rf ./miniconda
+  file_name="Miniconda3-py38_4.9.2-Linux-x86_64.sh"
+  wget https://repo.anaconda.com/miniconda/$file_name
+
+  # install miniconda in silent mode
+  bash $file_name -p ./miniconda -b
+
+  # activate conda
+  eval "$(./miniconda/bin/conda shell.bash hook)"
+}
+
+create_conda_env() {
+  python_version=$1
+  env_name=$2
+  conda create -n $env_name python=$python_version -y
+  conda activate $env_name
+  conda install conda-pack -y
+}
+
+create_python_backend_stub() {
+  rm -rf python_backend
+  git clone https://github.com/triton-inference-server/python_backend -b $PYTHON_BACKEND_REPO_TAG
+  (cd python_backend/ && mkdir builddir && cd builddir && \
+  cmake -DTRITON_BACKEND_REPO_TAG=$TRITON_BACKEND_REPO_TAG -DTRITON_COMMON_REPO_TAG=$TRITON_COMMON_REPO_TAG -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG ../ && \
+  make -j18 triton-python-backend-stub)
+}
diff --git a/qa/L0_backend_python/ensemble/ensemble_test.py b/qa/L0_backend_python/ensemble/ensemble_test.py
@@ -0,0 +1,83 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+sys.path.append("../../common")
+
+import test_util as tu
+import tritonclient.http as httpclient
+from tritonclient.utils import *
+import numpy as np
+import unittest
+
+
+class EnsembleTest(tu.TestResultCollector):
+    def test_ensemble(self):
+        model_name = "ensemble"
+        shape = [16]
+        with httpclient.InferenceServerClient("localhost:8000") as client:
+            input_data_0 = np.random.random(shape).astype(np.float32)
+            input_data_1 = np.random.random(shape).astype(np.float32)
+            inputs = [
+                httpclient.InferInput("INPUT0", input_data_0.shape,
+                                      np_to_triton_dtype(input_data_0.dtype)),
+                httpclient.InferInput("INPUT1", input_data_1.shape,
+                                      np_to_triton_dtype(input_data_1.dtype))
+            ]
+            inputs[0].set_data_from_numpy(input_data_0)
+            inputs[1].set_data_from_numpy(input_data_1)
+            result = client.infer(model_name, inputs)
+            output0 = result.as_numpy('OUTPUT0')
+            output1 = result.as_numpy('OUTPUT1')
+            self.assertIsNotNone(output0)
+            self.assertIsNotNone(output1)
+
+            self.assertTrue(np.allclose(output0, 2 * input_data_0))
+            self.assertTrue(np.allclose(output1, 2 * input_data_1))
+
+        model_name = "ensemble_gpu"
+        with httpclient.InferenceServerClient("localhost:8000") as client:
+            input_data_0 = np.random.random(shape).astype(np.float32)
+            input_data_1 = np.random.random(shape).astype(np.float32)
+            inputs = [
+                httpclient.InferInput("INPUT0", input_data_0.shape,
+                                      np_to_triton_dtype(input_data_0.dtype)),
+                httpclient.InferInput("INPUT1", input_data_1.shape,
+                                      np_to_triton_dtype(input_data_1.dtype))
+            ]
+            inputs[0].set_data_from_numpy(input_data_0)
+            inputs[1].set_data_from_numpy(input_data_1)
+            result = client.infer(model_name, inputs)
+            output0 = result.as_numpy('OUTPUT0')
+            output1 = result.as_numpy('OUTPUT1')
+            self.assertIsNotNone(output0)
+            self.assertIsNotNone(output1)
+
+            self.assertTrue(np.allclose(output0, 2 * input_data_0))
+            self.assertTrue(np.allclose(output1, 2 * input_data_1))
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/qa/L0_backend_python/ensemble/test.sh b/qa/L0_backend_python/ensemble/test.sh
@@ -0,0 +1,118 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+CLIENT_PY=./lifecycle_test.py
+CLIENT_LOG="./client.log"
+SERVER=/opt/tritonserver/bin/tritonserver
+BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1"
+PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG
+SERVER_ARGS=$BASE_SERVER_ARGS
+SERVER_LOG="./inference_server.log"
+EXPECTED_NUM_TESTS="1"
+REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
+DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"}
+RET=0
+
+source ../common.sh
+source ../../common/util.sh
+
+rm -rf models/ $CLIENT_LOG
+
+# Ensemble Model
+mkdir -p models/ensemble/1/
+cp ../../python_models/ensemble/config.pbtxt ./models/ensemble
+
+mkdir -p models/add_sub_1/1/
+cp ../../python_models/add_sub/config.pbtxt ./models/add_sub_1
+(cd models/add_sub_1 && \
+          sed -i "s/^name:.*/name: \"add_sub_1\"/" config.pbtxt)
+cp ../../python_models/add_sub/model.py ./models/add_sub_1/1/
+
+mkdir -p models/add_sub_2/1/
+cp ../../python_models/add_sub/config.pbtxt ./models/add_sub_2/
+(cd models/add_sub_2 && \
+          sed -i "s/^name:.*/name: \"add_sub_2\"/" config.pbtxt)
+cp ../../python_models/add_sub/model.py ./models/add_sub_2/1/
+
+# Ensemble GPU Model
+mkdir -p models/ensemble_gpu/1/
+cp ../../python_models/ensemble_gpu/config.pbtxt ./models/ensemble_gpu
+cp -r ${DATADIR}/qa_model_repository/libtorch_float32_float32_float32/ ./models
+(cd models/libtorch_float32_float32_float32 && \
+          echo "instance_group [ { kind: KIND_GPU }]" >> config.pbtxt)
+(cd models/libtorch_float32_float32_float32 && \
+          sed -i "s/^max_batch_size:.*/max_batch_size: 0/" config.pbtxt)
+(cd models/libtorch_float32_float32_float32 && \
+          sed -i "s/^version_policy:.*//" config.pbtxt)
+rm -rf models/libtorch_float32_float32_float32/2
+rm -rf models/libtorch_float32_float32_float32/3
+
+prev_num_pages=`get_shm_pages`
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    RET=1
+fi
+
+set +e
+python3 ensemble_test.py 2>&1 > $CLIENT_LOG
+
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** ensemble_test.py FAILED. \n***"
+    RET=1
+else
+    check_test_results $CLIENT_LOG $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
+fi
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+current_num_pages=`get_shm_pages`
+if [ $current_num_pages -ne $prev_num_pages ]; then
+    ls /dev/shm
+    echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
+Shared memory pages before starting triton equals to $prev_num_pages
+and shared memory pages after starting triton equals to $current_num_pages \n***"
+    RET=1
+fi
+
+if [ $RET -eq 1 ]; then
+    cat $CLIENT_LOG
+    echo -e "\n***\n*** Ensemble test FAILED. \n***"
+else
+    cat $CLIENT_LOG
+    echo -e "\n***\n*** Ensemble test PASSED. \n***"
+fi
+
+exit $RET
diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+CLIENT_PY=./lifecycle_test.py
+CLIENT_LOG="./client.log"
+source ../common.sh
+source ../../common/util.sh
+
+SERVER=/opt/tritonserver/bin/tritonserver
+BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1"
+PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG
+SERVER_ARGS=$BASE_SERVER_ARGS
+SERVER_LOG="./inference_server.log"
+REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
+DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"}
+
+RET=0
+
+rm -fr ./models
+rm -rf *.tar.gz
+apt update && apt install software-properties-common rapidjson-dev -y
+wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
+	gpg --dearmor - |  \
+	tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
+	apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \
+	apt-get update && \
+	apt-get install -y --no-install-recommends \
+	cmake-data=3.18.4-0kitware1ubuntu20.04.1 cmake=3.18.4-0kitware1ubuntu20.04.1
+install_conda
+
+# Create a model with python 3.9 version
+create_conda_env "3.9" "python-3-9"
+conda install numpy=1.20.1 -y
+create_python_backend_stub
+conda-pack -o python3.9.tar.gz
+path_to_conda_pack=`pwd`/python3.9.tar.gz
+mkdir -p models/python_3_9/1/
+cp ../../python_models/python_version/config.pbtxt ./models/python_3_9
+(cd models/python_3_9 && \
+          sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+cp ../../python_models/python_version/model.py ./models/python_3_9/1/
+cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9
+conda deactivate
+
+# Create a model with python 3.6 version
+create_conda_env "3.6" "python-3-6"
+conda install numpy=1.18.1 -y
+conda-pack -o python3.6.tar.gz
+path_to_conda_pack=`pwd`/python3.6.tar.gz
+create_python_backend_stub
+mkdir -p models/python_3_6/1/
+cp ../../python_models/python_version/config.pbtxt ./models/python_3_6
+(cd models/python_3_6 && \
+          sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
+cp ../../python_models/python_version/model.py ./models/python_3_6/1/
+cp python_backend/builddir/triton_python_backend_stub ./models/python_3_6
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+set +e
+grep "Python version is 3.6 and NumPy version is 1.18.1" $SERVER_LOG
+if [ $? -ne 0 ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Python 3.6 and NumPy 1.18.1 was not found in Triton logs. \n***"
+    RET=1
+fi
+
+grep "Python version is 3.9 and NumPy version is 1.20.1" $SERVER_LOG
+if [ $? -ne 0 ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Python 3.9 and NumPy 1.20.1 was not found in Triton logs. \n***"
+    RET=1
+fi
+set -e
+
+if [ $RET -eq 0 ]; then
+  echo -e "\n***\n*** Env Manager Test PASSED.\n***"
+else
+  cat $SERVER_LOG
+  echo -e "\n***\n*** Env Manager Test FAILED.\n***"
+fi
+
+exit $RET