[Servers] Add liveness and readiness probes (#3757)

* probes for ml-pipeline-ui * clean up comments * Use wget instead of curl, because wget is included in alpine * Also update marketplace manifest * Add readiness/liveness probe for api server * Add probes for python vis server
kubeflow · May 14, 2020 · f0c8eff · f0c8eff
1 parent 0f40372
commit f0c8eff
Show file tree

Hide file tree

Showing 5 changed files with 159 additions and 9 deletions.
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -78,7 +78,9 @@ COPY backend/src/apiserver/config/ /config
 COPY --from=compiler /samples/ /samples/
 
 # Adding CA certificate so API server can download pipeline through URL
-RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y ca-certificates \
+    # wget is used for liveness/readiness probe command
+    wget
 
 # Pin sample doc links to the commit that built the backend image
 RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_config.json && \

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml
@@ -570,6 +570,30 @@ spec:
           name: ml-pipeline-ui
           ports:
             - containerPort: 3000
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:3000/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:3000/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: ml-pipeline-ui
 ---
 apiVersion: apps/v1beta2
@@ -624,7 +648,32 @@ spec:
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-visualizationserver
           ports:
-            - containerPort: 8888
+            - name: http
+              containerPort: 8888
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:8888/
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:8888/
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: ml-pipeline-visualizationserver
 ---
 apiVersion: apps/v1beta2
@@ -694,8 +743,34 @@ spec:
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-api-server
           ports:
-            - containerPort: 8888
-            - containerPort: 8887
+            - name: http
+              containerPort: 8888
+            - name: grpc
+              containerPort: 8887
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:8888/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - -q # quiet
+                - -S # show server response
+                - -O
+                - "-" # Redirect output to stdout
+                - http://localhost:8888/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
       serviceAccountName: ml-pipeline
 ---
 apiVersion: v1

diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml
@@ -48,12 +48,36 @@ spec:
             configMapKeyRef:
               name: mysql-configmap
               key: port
-
-
         image: $(IMAGE_PREFIX)/api-server:$(IMAGE_TAG)
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-api-server
         ports:
-        - containerPort: 8888
-        - containerPort: 8887
+        - name: http
+          containerPort: 8888
+        - name: grpc
+          containerPort: 8887
+        readinessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
+        livenessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
       serviceAccountName: ml-pipeline
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml
@@ -26,4 +26,28 @@ spec:
                 fieldPath: metadata.namespace
           - name: ALLOW_CUSTOM_VISUALIZATIONS
             value: "true"
+        readinessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:3000/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
+        livenessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:3000/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
       serviceAccountName: ml-pipeline-ui
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-visualization-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-visualization-deployment.yaml
@@ -18,5 +18,30 @@ spec:
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-visualizationserver
         ports:
-        - containerPort: 8888
+        - name: http
+          containerPort: 8888
+        readinessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
+        livenessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
       serviceAccountName: ml-pipeline-visualizationserver