add tests for trt options (triton-inference-server#3315)

* add tests for trt options * plus fix * plus fix * Update test error message Co-authored-by: David Goodwin <davidg@nvidia.com>
Weizhongjin · Sep 3, 2021 · 8fbdf44 · 8fbdf44
1 parent ff88cb5
commit 8fbdf44
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/qa/L0_onnxtrt_optimization/test.sh b/qa/L0_onnxtrt_optimization/test.sh
@@ -43,6 +43,7 @@ CLIENT_LOG="./client.log"
 ONNXTRT_OPTIMIZATION_TEST=onnxtrt_optimization_test.py
 
 SERVER=/opt/tritonserver/bin/tritonserver
+CACHE_PATH=`pwd`/trt_cache
 SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --exit-on-error=false"
 SERVER_LOG="./inference_server.log"
 source ../common/util.sh
@@ -74,6 +75,15 @@ for MODEL in \
             parameters { key: \"precision_mode\" value: \"FP16\" } \
             parameters { key: \"max_workspace_size_bytes\" value: \"1073741824\" } }]}}" \
             >> config.pbtxt) && \
+    # GPU execution accelerators with cache enabled
+    cp -r models/${MODEL}_test models/${MODEL}_cache_on && \
+    (cd models/${MODEL}_cache_on && \
+            sed -i 's/_float32_test/_float32_cache_on/' \
+                config.pbtxt && \
+            echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\" \
+            parameters { key: \"trt_engine_cache_enable\" value: \"1\" } \
+            parameters { key: \"trt_engine_cache_path\" value: \"${CACHE_PATH}\" } }]}}" \
+            >> config.pbtxt) && \
     # GPU execution accelerators with unknown parameters
     cp -r models/${MODEL}_test models/${MODEL}_unknown_param && \
     (cd models/${MODEL}_unknown_param && \
@@ -120,6 +130,12 @@ for MODEL in \
         RET=1
     fi
 
+    grep "TensorRT Execution Accelerator is set for '${MODEL}_cache_on'" $SERVER_LOG
+    if [ $? -ne 0 ]; then
+        echo -e "\n***\n*** Failed. Expected TensorRT Execution Accelerator is set for '${MODEL}_cache_on'\n***"
+        RET=1
+    fi
+
     grep "failed to load '${MODEL}_unknown_param' version 1: Invalid argument: unknown parameter 'segment_size' is provided for TensorRT Execution Accelerator" $SERVER_LOG
     if [ $? -ne 0 ]; then
         echo -e "\n***\n*** Failed. Expected unknown parameter 'segment_size' returns error\n***"

diff --git a/qa/L0_perf_resnet/test.sh b/qa/L0_perf_resnet/test.sh
@@ -53,6 +53,8 @@ TFAMP_MODEL_NAME="resnet50v1.5_fp16_savedmodel_amp"
 ARCH=${ARCH:="x86_64"}
 REPODIR=${REPODIR:="/data/inferenceserver/${REPO_VERSION}"}
 TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"}
+CACHE_PATH=`pwd`/trt_cache
+
 
 #
 # Test minimum latency
@@ -92,6 +94,8 @@ for MODEL_NAME in $OPTIMIZED_MODEL_NAMES; do
         if [ "${MODEL_NAME}" = "${ONNXTRT_MODEL_NAME}" ] ; then
             echo "parameters { key: \"precision_mode\" value: \"FP16\" }" >> ${CONFIG_PATH}
             echo "parameters { key: \"max_workspace_size_bytes\" value: \"1073741824\" }" >> ${CONFIG_PATH}
+            echo "parameters { key: \"trt_engine_cache_enable\" value: \"1\" }" >> ${CONFIG_PATH}
+            echo "parameters { key: \"trt_engine_cache_path\" value: \"${CACHE_PATH}\" } " >> ${CONFIG_PATH}
         fi
     fi
     echo "} ]" >> ${CONFIG_PATH}