Ps ci perf

.circleci/config.yml

@@ -210,7 +210,4 @@ workflows:
           name: performance-regression-<< matrix.executor >>
           requires:
             - build-<< matrix.executor >>
-          matrix:
-            <<: *matrix
-            exclude:
-              - executor: "py27"
+          matrix: *matrix

tests/performance/tests/api_description/api_description.yaml

@@ -16,10 +16,12 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10s"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s

tests/performance/tests/batch_and_single_inference/batch_and_single_inference.yaml

@@ -17,6 +17,7 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
@@ -28,6 +29,7 @@ services:
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=2&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring # should be added in modules section
     ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor

tests/performance/tests/batch_inference/batch_inference.yaml

@@ -17,6 +17,7 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
@@ -26,6 +27,7 @@ services:
       - "curl -s -X PUT  http://localhost:8081/models/resnet-152?min_worker=2&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring # should be added in modules section
     ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor

tests/performance/tests/examples_local_criteria/examples_local_criteria.yaml

@@ -17,11 +17,13 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring # should be added in modules section
     ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor

tests/performance/tests/examples_local_monitoring/examples_local_monitoring.yaml

@@ -16,11 +16,13 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring # should be added in modules section
     ServerLocalClient: # keyword from metrics_monitoring_inproc.Monitor

tests/performance/tests/examples_remote_criteria/examples_remote_criteria.yaml

@@ -12,11 +12,13 @@ scenarios:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: monitoring
     server-agent:

tests/performance/tests/examples_remote_monitoring/examples_remote_monitoring.yaml

@@ -14,11 +14,13 @@ scenarios:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: monitoring
     server-agent:

tests/performance/tests/examples_starter/examples_starter.yaml

@@ -11,10 +11,12 @@ scenarios:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
 

tests/performance/tests/examples_unregister_while_inferencing/examples_unregister_while_inferencing.yaml

@@ -16,6 +16,7 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "./tests/unregister_while_inferencing/custom_model_for_slow_inference.sh build"
       - "multi-model-server --start --model-store=tmp_store > /dev/null 2>&1"
@@ -24,6 +25,7 @@ services:
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "./tests/unregister_while_inferencing/custom_model_for_slow_inference.sh clean"
       - "rm kitten.jpg"
   - module: server_local_monitoring

tests/performance/tests/health_check/health_check.yaml

@@ -16,10 +16,12 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s

tests/performance/tests/inference_multiple_models/inference_multiple_models.yaml

@@ -16,6 +16,7 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
@@ -25,6 +26,7 @@ services:
       - "curl -s -X PUT  http://localhost:8081/models/resnet-18?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/inference_multiple_worker/inference_multiple_worker.yaml

@@ -17,13 +17,15 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=4&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/inference_single_worker/inference_single_worker.yaml

@@ -17,13 +17,15 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/list_models/list_models.yaml

@@ -16,12 +16,14 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/shufflenet.mar"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s

tests/performance/tests/model_description/model_description.yaml

@@ -16,12 +16,14 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s

tests/performance/tests/multiple_inference_and_scaling/multiple_inference_and_scaling.yaml

@@ -16,6 +16,7 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
@@ -25,6 +26,7 @@ services:
       - "curl -s -X PUT  http://localhost:8081/models/resnet-18?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/register_unregister/register_unregister.yaml

@@ -17,11 +17,13 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 10s"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/register_unregister_multiple/environments/ci_linux_medium.yaml

@@ -9,5 +9,5 @@ settings:
     UNREG_RT: 150ms
     TOTAL_PROCS_WITH_EXTRA_MODEL_AND_WORKERS : 5
     TOTAL_PROCS_WITHOUT_EXTRA_MODEL : 2
-    TOTAL_FDS : 509
+    TOTAL_FDS : 524
     FRNTEND_MEM : 425000000 #425MB

tests/performance/tests/register_unregister_multiple/register_unregister_multiple.yaml

@@ -16,12 +16,14 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "curl -s -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/resnet-18.mar"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
       - "rm kitten.jpg"
   - module: server_local_monitoring
     ServerLocalClient:

tests/performance/tests/scale_down_workers/environments/ci_linux_medium.yaml

@@ -9,7 +9,7 @@ settings:
     TOTAL_WRKRS_AFTR_SCL_DWN  : 2
     FRNTEND_FDS : 494
     TOTAL_WRKRS_FDS_B4_SCL_DWN: 43
-    TOTAL_WRKRS_FDS_AFTR_SCL_DWN: 24
+    TOTAL_WRKRS_FDS_AFTR_SCL_DWN: 17
     FRNTEND_MEM : 770000000 #770MB
     TOTAL_WRKRS_MEM_B4_SCL_DWN : 430000000 #430MB
-    TOTAL_WRKRS_MEM_AFTR_SCL_DWN : 220000000 #220MB
+    TOTAL_WRKRS_MEM_AFTR_SCL_DWN : 200000000 #200MB

tests/performance/tests/scale_down_workers/scale_down_workers.yaml

@@ -16,12 +16,14 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=4&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s

tests/performance/tests/scale_up_workers/scale_up_workers.yaml

@@ -16,12 +16,14 @@ modules:
 services:
   - module: shellexec
     prepare:
+      - "ps aux | grep '[c]om.amazonaws.ml.mms.ModelServer' | awk '{print $2}' | xargs kill -9"
       - "multi-model-server --start > /dev/null 2>&1"
       - "sleep 20s"
       - "curl -s -X POST http://localhost:8081/models?url=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar"
       - "curl -s -X PUT  http://localhost:8081/models/squeezenet_v1.1?min_worker=1&synchronous=true"
     post-process:
       - "multi-model-server --stop > /dev/null 2>&1"
+      - "sleep 10"
   - module: server_local_monitoring
     ServerLocalClient:
       - interval: 1s