e2e: e2e offline test support multiple python version (#1837)

* multiple python

* modify torch related version to fit pfp example

* remove ucf 101 runtime, add python-dev in e2e base image

* modify runtime outside e2e

* fix ut error

* fix simple e2e error

* when controller container is starting logs would fail

* change restart policy in pod to Never

* fix prepare

* fix review issue

* enlarge controller max heap size

* add pip cache ; update notification message

docker/Dockerfile.base_e2e

@@ -40,7 +40,7 @@ RUN apt-get install -y openjdk-11-jdk \
 # Install python
 RUN add-apt-repository -y ppa:deadsnakes/ppa \
     && apt-get update \
-    && apt-get install -y python3.7 python3.8 python3.9 python3.10 python3-pip python3-venv python3.8-venv python3.7-venv python3.9-venv python3.10-venv \
+    && apt-get install -y python3.7 python3.7-dev python3.8 python3.8-dev python3.9 python3.9-dev python3.10 python3.10-dev python3-pip python3-venv python3.8-venv python3.7-venv python3.9-venv python3.10-venv \
     && apt-get clean all \
     && rm -rf /var/lib/apt/lists/* /tmp/*
 

docker/charts/templates/controller-deployment.yaml

@@ -67,7 +67,7 @@ spec:
               value: controller
               {{- if not .Values.minikube.enabled }}
             - name: JVM_XMX
-              value: {{ .Values.resources.controller.requests.memory }}
+              value: {{ .Values.resources.controller.limits.memory }}
               {{- end }}
             - name: SW_CONTROLLER_PORT
               value: "{{ .Values.controller.containerPort }}"

example/runtime/pytorch-cn-mirror/requirements.txt

@@ -9,19 +9,14 @@ pycocotools
 opencv-python-headless
 # fixed versions
 numpy==1.21.6; python_version=='3.7'
-torch==1.11.0; python_version=='3.7'
-torchaudio==0.11.0; python_version=='3.7'
-torchdata==0.3.0; python_version=='3.7'
-torchtext==0.12.0; python_version=='3.7'
-torchvision==0.12.0; python_version=='3.7'
 scikit-learn==1.0.2; python_version=='3.7'
 scipy==1.7.3; python_version=='3.7'
 numpy==1.23.2; python_version>='3.8'
-torch==1.12.1; python_version>='3.8'
 scikit-learn==1.1.1; python_version>='3.8'
 scipy==1.8.1; python_version>='3.8'
 importlib-metadata>=4.0.0, <=4.2.0;python_version < '3.8'
-torchaudio==0.12.1; python_version>='3.8'
-torchdata==0.4.1; python_version>='3.8'
-torchtext==0.13.1; python_version>='3.8'
-torchvision==0.13.1; python_version>='3.8'
+torch==1.12.1
+torchaudio==0.12.1
+torchdata==0.4.1
+torchtext==0.13.1
+torchvision==0.13.1

example/runtime/pytorch-e2e/requirements.txt

@@ -0,0 +1,17 @@
+# for pfp
+pycocotools
+# for ucf101
+opencv-python-headless
+# fixed versions
+numpy==1.21.6; python_version=='3.7'
+scikit-learn==1.0.2; python_version=='3.7'
+scipy==1.7.3; python_version=='3.7'
+numpy==1.23.2; python_version>='3.8'
+scikit-learn==1.1.1; python_version>='3.8'
+scipy==1.8.1; python_version>='3.8'
+importlib-metadata>=4.0.0, <=4.2.0;python_version < '3.8'
+torch==1.12.1
+torchaudio==0.12.1
+torchdata==0.4.1
+torchtext==0.13.1
+torchvision==0.13.1

example/runtime/pytorch-e2e/runtime-3-10.yaml

@@ -0,0 +1,19 @@
+api_version: 1.1
+dependencies:
+  - pip:
+      - Pillow
+      - numpy
+      - scikit-learn
+      - torchvision
+      - torch
+      - torchdata
+      - torchtext
+      - torchaudio
+      - pycocotools
+  - requirements.txt
+environment:
+  python: "3.10"
+  arch: noarch
+  os: ubuntu:20.04
+mode: venv
+name: pytorch

example/runtime/pytorch-e2e/runtime-3-7.yaml

@@ -0,0 +1,19 @@
+api_version: 1.1
+dependencies:
+  - pip:
+      - Pillow
+      - numpy
+      - scikit-learn
+      - torchvision
+      - torch
+      - torchdata
+      - torchtext
+      - torchaudio
+      - pycocotools
+  - requirements.txt
+environment:
+  arch: noarch
+  os: ubuntu:20.04
+  python: 3.7
+mode: venv
+name: pytorch

example/runtime/pytorch-e2e/runtime-3-8.yaml

@@ -0,0 +1,19 @@
+api_version: 1.1
+dependencies:
+  - pip:
+      - Pillow
+      - numpy
+      - scikit-learn
+      - torchvision
+      - torch
+      - torchdata
+      - torchtext
+      - torchaudio
+      - pycocotools
+  - requirements.txt
+environment:
+  arch: noarch
+  os: ubuntu:20.04
+  python: 3.8
+mode: venv
+name: pytorch

example/runtime/pytorch-e2e/runtime-3-9.yaml

@@ -0,0 +1,19 @@
+api_version: 1.1
+dependencies:
+  - pip:
+      - Pillow
+      - numpy
+      - scikit-learn
+      - torchvision
+      - torch
+      - torchdata
+      - torchtext
+      - torchaudio
+      - pycocotools
+  - requirements.txt
+environment:
+  python: 3.9
+  arch: noarch
+  os: ubuntu:20.04
+mode: venv
+name: pytorch

example/runtime/pytorch-e2e/runtime.yaml

@@ -16,7 +16,7 @@ dependencies:
       - torchtext
       - torchaudio
       - pycocotools
-  - requirements-sw-lock.txt
+  - requirements.txt
 environment:
   arch: noarch
   os: ubuntu:20.04

example/runtime/pytorch/requirements.txt

@@ -5,19 +5,14 @@ pycocotools
 opencv-python-headless
 # fixed versions
 numpy==1.21.6; python_version=='3.7'
-torch==1.11.0; python_version=='3.7'
-torchaudio==0.11.0; python_version=='3.7'
-torchdata==0.3.0; python_version=='3.7'
-torchtext==0.12.0; python_version=='3.7'
-torchvision==0.12.0; python_version=='3.7'
 scikit-learn==1.0.2; python_version=='3.7'
 scipy==1.7.3; python_version=='3.7'
 numpy==1.23.2; python_version>='3.8'
-torch==1.12.1; python_version>='3.8'
 scikit-learn==1.1.1; python_version>='3.8'
 scipy==1.8.1; python_version>='3.8'
 importlib-metadata>=4.0.0, <=4.2.0;python_version < '3.8'
-torchaudio==0.12.1; python_version>='3.8'
-torchdata==0.4.1; python_version>='3.8'
-torchtext==0.13.1; python_version>='3.8'
-torchvision==0.13.1; python_version>='3.8'
+torch==1.12.1
+torchaudio==0.12.1
+torchdata==0.4.1
+torchtext==0.13.1
+torchvision==0.13.1