diff --git a/qa/L0_backend_python/argument_validation/models/argument_validation/1/model.py b/qa/L0_backend_python/argument_validation/models/argument_validation/1/model.py new file mode 100644 index 0000000000..5af497aa0b --- /dev/null +++ b/qa/L0_backend_python/argument_validation/models/argument_validation/1/model.py @@ -0,0 +1,176 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import numpy as np +import unittest +import triton_python_backend_utils as pb_utils + + +class ArgumentValidationTest(unittest.TestCase): + + def test_infer_request_args(self): + # Dummy arguments used in the tests. + inputs = [pb_utils.Tensor('INPUT0', np.asarray([1, 2], dtype=np.int32))] + model_name = 'my_model' + requested_output_names = ['my_output'] + + # + # inputs field validation + # + + # Test list of None as inputs + with self.assertRaises(pb_utils.TritonModelException) as e: + pb_utils.InferenceRequest( + inputs=[None], + model_name=model_name, + requested_output_names=requested_output_names) + + # Test None object as list of inputs + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + inputs=None, + model_name=model_name, + requested_output_names=requested_output_names) + + # model_name validation + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + model_name=None, + inputs=inputs, + requested_output_names=requested_output_names) + + # + # Requested output name validations + # + + # Test list of None objects as requested_output_names + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest(requested_output_names=[None], + inputs=inputs, + model_name=model_name) + + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest(requested_output_names=None, + inputs=inputs, + model_name=model_name) + + # Other arguments validation + + # correlation_id set to None + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + requested_output_names=requested_output_names, + inputs=inputs, + model_name=model_name, + correleation_id=None) + + # request_id set to None + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + requested_output_names=requested_output_names, + inputs=inputs, + model_name=model_name, + request_id=None) + + # model_version set to None + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + requested_output_names=requested_output_names, + inputs=inputs, + model_name=model_name, + model_version=None) + + # flags set to None + with self.assertRaises(TypeError) as e: + pb_utils.InferenceRequest( + requested_output_names=requested_output_names, + inputs=inputs, + model_name=model_name, + flags=None) + + # Empty lists should not raise an exception + pb_utils.InferenceRequest(requested_output_names=[], + inputs=[], + model_name=model_name) + + def test_infer_response_args(self): + outputs = [ + pb_utils.Tensor('OUTPUT0', np.asarray([1, 2], dtype=np.int32)) + ] + + # Test list of None object as output tensor + with self.assertRaises(pb_utils.TritonModelException) as e: + pb_utils.InferenceResponse(output_tensors=[None]) + + # Test None as output tensors + with self.assertRaises(TypeError) as e: + pb_utils.InferenceResponse(output_tensors=None) + + # This should not raise an exception + pb_utils.InferenceResponse(output_tensors=[]) + pb_utils.InferenceResponse(outputs) + + def test_tensor_args(self): + np_array = np.asarray([1, 2], dtype=np.int32) + + # Test None as tensor name + with self.assertRaises(TypeError) as e: + pb_utils.Tensor(None, np_array) + + # Test None as Numpy array + with self.assertRaises(TypeError) as e: + pb_utils.Tensor("OUTPUT0", None) + + # Test None as dlpack capsule + with self.assertRaises(TypeError) as e: + pb_utils.Tensor.from_dlpack("OUTPUT0", None) + + # Test empty string as model name (from_dlpack) + with self.assertRaises(TypeError) as e: + pb_utils.Tensor.from_dlpack("", None) + + # Test empty string as model name + with self.assertRaises(TypeError) as e: + pb_utils.Tensor("", None) + + +class TritonPythonModel: + """This model tests the Python API arguments to make sure invalid args are + rejected.""" + + def execute(self, requests): + responses = [] + for _ in requests: + # Run the unittest and store the results in InferenceResponse. + test = unittest.main('model', exit=False) + responses.append( + pb_utils.InferenceResponse([ + pb_utils.Tensor( + 'OUTPUT0', + np.array([test.result.wasSuccessful()], + dtype=np.float16)) + ])) + return responses diff --git a/qa/L0_backend_python/argument_validation/models/argument_validation/config.pbtxt b/qa/L0_backend_python/argument_validation/models/argument_validation/config.pbtxt new file mode 100644 index 0000000000..4c02983319 --- /dev/null +++ b/qa/L0_backend_python/argument_validation/models/argument_validation/config.pbtxt @@ -0,0 +1,39 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +name: "argument_validation" +backend: "python" +max_batch_size: 0 + +output [ + { + name: "OUTPUT0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] + +instance_group [{ kind: KIND_CPU }] diff --git a/qa/L0_backend_python/argument_validation/test.sh b/qa/L0_backend_python/argument_validation/test.sh new file mode 100644 index 0000000000..f80ce3e84b --- /dev/null +++ b/qa/L0_backend_python/argument_validation/test.sh @@ -0,0 +1,77 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +CLIENT_PY=../python_unittest.py +CLIENT_LOG="./client.log" +EXPECTED_NUM_TESTS="1" +TEST_RESULT_FILE='test_results.txt' +TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} +SERVER=${TRITON_DIR}/bin/tritonserver +BACKEND_DIR=${TRITON_DIR}/backends +SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" +SERVER_LOG="./inference_server.log" + +RET=0 +source ../../common/util.sh + +rm -fr *.log + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + RET=1 +fi + +set +e +export MODEL_NAME="argument_validation" +python3 $CLIENT_PY > $CLIENT_LOG 2>&1 + +if [ $? -ne 0 ]; then + echo -e "\n***\n*** python_unittest.py FAILED. \n***" + RET=1 +else + check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Result Verification Failed\n***" + RET=1 + fi +fi +set -e + +kill $SERVER_PID +wait $SERVER_PID + +if [ $RET -eq 1 ]; then + cat $CLIENT_LOG + cat $SERVER_LOG + echo -e "\n***\n*** Argument validation test FAILED. \n***" +else + echo -e "\n***\n*** Argument validation test PASSED. \n***" +fi + +exit $RET diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 664ae852a4..f5e9391bb1 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -391,6 +391,11 @@ if [ $? -ne 0 ]; then RET=1 fi +(cd argument_validation && bash -ex test.sh) +if [ $? -ne 0 ]; then + RET=1 +fi + if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else