Fix runtime for L0_client_memory_growth (triton-inference-server#3190)

* Update repetition * Update comments * Correct test duration * Update comments * Update repetition Co-authored-by: Kris Hung <krish@krish-dt.nvidia.com>
ileixe · Aug 5, 2021 · a7c344a · a7c344a
1 parent cff2874
commit a7c344a
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 4 deletions.
diff --git a/qa/L0_client_memory_growth/test.sh b/qa/L0_client_memory_growth/test.sh
@@ -56,8 +56,12 @@ source ../common/util.sh
 # Set the number of repetitions in nightly and weekly tests
 # Set the email subject for nightly and weekly tests
 if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
-    REPETITION_CPP=2000000
-    REPETITION_PY=2400000
+    # Run the test for each case approximately 1.5 hours
+    # All tests are run cumulatively for 7 hours
+    REPETITION_HTTP_CPP=1300000
+    REPETITION_HTTP_PY=2100000
+    REPETITION_GRPC_CPP=10000000
+    REPETITION_GRPC_PY=1500000
     EMAIL_SUBJECT="Weekly"
 else
     REPETITION_CPP=100000
@@ -78,6 +82,16 @@ for PROTOCOL in http grpc; do
         MASSIF_LOG="./${PROTOCOL}.${LANG}.massif"
         LEAKCHECK_ARGS="$LEAKCHECK_ARGS_BASE --log-file=$LEAKCHECK_LOG --massif-out-file=$MASSIF_LOG"
 
+        if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
+            if [ $PROTOCOL ==  http ]; then
+                REPETITION_CPP=$REPETITION_HTTP_CPP
+                REPETITION_PY=$REPETITION_HTTP_PY
+            else
+                REPETITION_CPP=$REPETITION_GRPC_CPP
+                REPETITION_PY=$REPETITION_GRPC_PY
+            fi
+        fi 
+
         run_server
         if [ "$SERVER_PID" == "0" ]; then
             echo -e "\n***\n*** Failed to start $SERVER\n***"

diff --git a/qa/L0_memory_growth/test.sh b/qa/L0_memory_growth/test.sh
@@ -73,6 +73,8 @@ CLIENT_BS=8
 # Set the number of repetitions in nightly and weekly tests
 # Set the email subject for nightly and weekly tests
 if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
+    # Run the test for each model approximately 1.5 hours
+    # All tests are run cumulatively for 7 hours
     REPETITION=200
     EMAIL_SUBJECT="Weekly"
 else
@@ -141,17 +143,17 @@ for MODEL in $(ls models); do
 
     set +e
 
+    SECONDS=0
     # Run the perf analyzer 'REPETITION' times
     for ((i=1; i<=$REPETITION; i++)); do
-        SECONDS=0
         $PERF_ANALYZER -v -m $MODEL -i grpc --concurrency-range $CONCURRENCY -b $CLIENT_BS >> $CLIENT_LOG 2>&1
-        TEST_DURATION=$SECONDS
         if [ $? -ne 0 ]; then
             cat $CLIENT_LOG
             echo -e "\n***\n*** perf_analyzer for $MODEL failed on iteration $i\n***"
             RET=1   
         fi
     done
+    TEST_DURATION=$SECONDS
 
     set -e