From a7c344a9d162f178cc5a226337c3ef593d2cb220 Mon Sep 17 00:00:00 2001
From: Kris Hung <krishung0521@gmail.com>
Date: Thu, 5 Aug 2021 09:48:46 -0700
Subject: [PATCH] Fix runtime for L0_client_memory_growth (#3190)

* Update repetition

* Update comments

* Correct test duration

* Update comments

* Update repetition

Co-authored-by: Kris Hung <krish@krish-dt.nvidia.com>
---
 qa/L0_client_memory_growth/test.sh | 18 ++++++++++++++++--
 qa/L0_memory_growth/test.sh        |  6 ++++--
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/qa/L0_client_memory_growth/test.sh b/qa/L0_client_memory_growth/test.sh
index 245e184077..0bd4358132 100755
--- a/qa/L0_client_memory_growth/test.sh
+++ b/qa/L0_client_memory_growth/test.sh
@@ -56,8 +56,12 @@ source ../common/util.sh
 # Set the number of repetitions in nightly and weekly tests
 # Set the email subject for nightly and weekly tests
 if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
-    REPETITION_CPP=2000000
-    REPETITION_PY=2400000
+    # Run the test for each case approximately 1.5 hours
+    # All tests are run cumulatively for 7 hours
+    REPETITION_HTTP_CPP=1300000
+    REPETITION_HTTP_PY=2100000
+    REPETITION_GRPC_CPP=10000000
+    REPETITION_GRPC_PY=1500000
     EMAIL_SUBJECT="Weekly"
 else
     REPETITION_CPP=100000
@@ -78,6 +82,16 @@ for PROTOCOL in http grpc; do
         MASSIF_LOG="./${PROTOCOL}.${LANG}.massif"
         LEAKCHECK_ARGS="$LEAKCHECK_ARGS_BASE --log-file=$LEAKCHECK_LOG --massif-out-file=$MASSIF_LOG"
 
+        if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
+            if [ $PROTOCOL ==  http ]; then
+                REPETITION_CPP=$REPETITION_HTTP_CPP
+                REPETITION_PY=$REPETITION_HTTP_PY
+            else
+                REPETITION_CPP=$REPETITION_GRPC_CPP
+                REPETITION_PY=$REPETITION_GRPC_PY
+            fi
+        fi 
+        
         run_server
         if [ "$SERVER_PID" == "0" ]; then
             echo -e "\n***\n*** Failed to start $SERVER\n***"
diff --git a/qa/L0_memory_growth/test.sh b/qa/L0_memory_growth/test.sh
index 25825c4720..084bc25dd8 100755
--- a/qa/L0_memory_growth/test.sh
+++ b/qa/L0_memory_growth/test.sh
@@ -73,6 +73,8 @@ CLIENT_BS=8
 # Set the number of repetitions in nightly and weekly tests
 # Set the email subject for nightly and weekly tests
 if [ "$TRITON_PERF_WEEKLY" == 1 ]; then
+    # Run the test for each model approximately 1.5 hours
+    # All tests are run cumulatively for 7 hours
     REPETITION=200
     EMAIL_SUBJECT="Weekly"
 else
@@ -141,17 +143,17 @@ for MODEL in $(ls models); do
 
     set +e
 
+    SECONDS=0
     # Run the perf analyzer 'REPETITION' times
     for ((i=1; i<=$REPETITION; i++)); do
-        SECONDS=0
         $PERF_ANALYZER -v -m $MODEL -i grpc --concurrency-range $CONCURRENCY -b $CLIENT_BS >> $CLIENT_LOG 2>&1
-        TEST_DURATION=$SECONDS
         if [ $? -ne 0 ]; then
             cat $CLIENT_LOG
             echo -e "\n***\n*** perf_analyzer for $MODEL failed on iteration $i\n***"
             RET=1   
         fi
     done
+    TEST_DURATION=$SECONDS
 
     set -e