Upgraded Argo to v2.7.5 (#3537)

* Upgraded Argo to v2.7.4 * Downgraded the Argo CLI version to 2.4.3 See argoproj/argo-workflows#2793 * Removed the argo cli arg that had been removed * Updated to Argo 2.7.5 * Added workflowtemplates and cronworkflows to the Role * Added the new Argo CRDs
kubeflow · May 12, 2020 · e83bd7d · e83bd7d
1 parent bd8c1dd
commit e83bd7d
Show file tree

Hide file tree

Showing 15 changed files with 139 additions and 27 deletions.
diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
@@ -211,10 +211,10 @@ steps:
   args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
   id:   'pullCloudsqlProxy'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
   id:   'pullArgoWorkflowController'
 
 # Tag for Hosted - SemVersion to Major.Minor parsing

diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml
@@ -438,14 +438,14 @@ steps:
     docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplace'
   waitFor: ['pullArgoExecutor']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
   id:   'tagArgoExecutorForMarketplaceTest'
   waitFor: ['pullArgoExecutor']
 - id:   'tagArgoExecutorForMarketplaceMajorMinor'
@@ -455,20 +455,20 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
   id:   'pullArgoWorkflowController'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplace'
   waitFor: ['pullArgoWorkflowController']
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
+  args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
   id:   'tagArgoWorkflowControllerForMarketplaceTest'
   waitFor: ['pullArgoWorkflowController']
 - id:   'tagArgoWorkflowControllerForMarketplaceMajorMinor'
@@ -478,8 +478,8 @@ steps:
   args:
   - -ceux
   - |
-    docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
-    docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
+    docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
     docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
 

diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -32,7 +32,8 @@ COPY backend/requirements.txt .
 RUN python3 -m pip install -r requirements.txt
 
 # Downloading Argo CLI so that the samples are validated
-ADD  https://github.com/argoproj/argo/releases/download/v2.3.0/argo-linux-amd64 /usr/local/bin/argo
+#ADD  https://github.com/argoproj/argo/releases/download/v2.7.5/argo-linux-amd64 /usr/local/bin/argo
+ADD  https://github.com/argoproj/argo/releases/download/v2.4.3/argo-linux-amd64 /usr/local/bin/argo
 RUN chmod +x /usr/local/bin/argo
 
 WORKDIR /go/src/github.com/kubeflow/pipelines

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml
@@ -14,6 +14,56 @@ spec:
   scope: Namespaced
   version: v1alpha1
 ---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterworkflowtemplates.argoproj.io
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  group: argoproj.io
+  names:
+    kind: ClusterWorkflowTemplate
+    plural: clusterworkflowtemplates
+    shortNames:
+    - clusterwftmpl
+    - cwft
+  scope: Cluster
+  version: v1alpha1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: cronworkflows.argoproj.io
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  group: argoproj.io
+  names:
+    kind: CronWorkflow
+    plural: cronworkflows
+    shortNames:
+    - cronwf
+    - cwf
+  scope: Namespaced
+  version: v1alpha1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: workflowtemplates.argoproj.io
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  group: argoproj.io
+  names:
+    kind: WorkflowTemplate
+    plural: workflowtemplates
+    shortNames:
+    - wftmpl
+  scope: Namespaced
+  version: v1alpha1
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -61,6 +111,9 @@ rules:
     resources:
       - workflows
       - workflows/finalizers
+      - workflowtemplates
+      - workflowtemplates/finalizers
+      - cronworkflows
     verbs:
       - get
       - list

diff --git a/manifests/kustomize/base/argo/cluster-scoped/workflow-crd.yaml b/manifests/kustomize/base/argo/cluster-scoped/workflow-crd.yaml
@@ -10,4 +10,58 @@ spec:
     kind: Workflow
     plural: workflows
     shortNames:
-    - wf
+    - wf
+  additionalPrinterColumns:
+  - JSONPath: .status.phase
+    description: Status of the workflow
+    name: Status
+    type: string
+  - JSONPath: .status.startedAt
+    description: When the workflow was started
+    format: date-time
+    name: Age
+    type: date
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterworkflowtemplates.argoproj.io
+spec:
+  group: argoproj.io
+  names:
+    kind: ClusterWorkflowTemplate
+    plural: clusterworkflowtemplates
+    shortNames:
+    - clusterwftmpl
+    - cwft
+  scope: Cluster
+  version: v1alpha1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: cronworkflows.argoproj.io
+spec:
+  group: argoproj.io
+  names:
+    kind: CronWorkflow
+    plural: cronworkflows
+    shortNames:
+    - cronwf
+    - cwf
+  scope: Namespaced
+  version: v1alpha1
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: workflowtemplates.argoproj.io
+spec:
+  group: argoproj.io
+  names:
+    kind: WorkflowTemplate
+    plural: workflowtemplates
+    shortNames:
+    - wftmpl
+  scope: Namespaced
+  version: v1alpha1
diff --git a/manifests/kustomize/base/argo/workflow-controller-configmap.yaml b/manifests/kustomize/base/argo/workflow-controller-configmap.yaml
@@ -6,7 +6,7 @@ data:
   config: |
     {
     namespace: $(NAMESPACE),
-    executorImage: gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance,
+    executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
     artifactRepository:
     {
         s3: {

diff --git a/manifests/kustomize/base/argo/workflow-controller-deployment.yaml b/manifests/kustomize/base/argo/workflow-controller-deployment.yaml
@@ -27,7 +27,7 @@ spec:
         - --configmap
         - workflow-controller-configmap
         - --executor-image
-        - gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance
+        - gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance
         command:
         - workflow-controller
         env:
@@ -36,7 +36,7 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
-        image: gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance
+        image: gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance
         imagePullPolicy: IfNotPresent
         name: workflow-controller
         resources: {}

diff --git a/manifests/kustomize/base/argo/workflow-controller-role.yaml b/manifests/kustomize/base/argo/workflow-controller-role.yaml
@@ -36,6 +36,9 @@ rules:
   resources:
   - workflows
   - workflows/finalizers
+  - workflowtemplates
+  - workflowtemplates/finalizers
+  - cronworkflows
   verbs:
   - get
   - list

diff --git a/test/check-argo-status.sh b/test/check-argo-status.sh
@@ -50,7 +50,7 @@ if [[ "$workflow_completed" == "false" ]] || [[ "$workflow_failed" != "false" ]]
   fi
 
   echo "=========Argo Workflow Logs========="
-  argo logs -w "${ARGO_WORKFLOW}" -n "${NAMESPACE}"
+  argo logs "${ARGO_WORKFLOW}" -n "${NAMESPACE}"
 
   echo "========All workflows============="
 

diff --git a/test/install-argo.sh b/test/install-argo.sh
@@ -25,7 +25,7 @@ ACCOUNT=$(gcloud info --format='value(config.account)')
 kubectl create clusterrolebinding PROW_BINDING --clusterrole=cluster-admin --user=$ACCOUNT --dry-run -o yaml | kubectl apply -f -
 kubectl create clusterrolebinding DEFAULT_BINDING --clusterrole=cluster-admin --serviceaccount=default:default --dry-run -o yaml | kubectl apply -f -
 
-ARGO_VERSION=v2.3.0
+ARGO_VERSION=v2.7.5
 
 # if argo is not installed
 if ! which argo; then

diff --git a/test/sample-test/Dockerfile b/test/sample-test/Dockerfile
@@ -22,7 +22,8 @@ RUN cd /python/src/github.com/kubeflow/pipelines
 # Install argo.
 RUN echo "install argo"
 RUN mkdir -p /usr/local/bin/
-RUN ARGO_VERSION=v2.3.0 && curl -sSL -o /usr/local/bin/argo \
+#RUN ARGO_VERSION=v2.7.5 && curl -sSL -o /usr/local/bin/argo \
+RUN ARGO_VERSION=v2.4.3 && curl -sSL -o /usr/local/bin/argo \
     "https://github.com/argoproj/argo/releases/download/$ARGO_VERSION/argo-linux-amd64" && \
     chmod +x /usr/local/bin/argo
 ENV PATH $PATH:/usr/local/bin/argo

diff --git a/test/tag_for_hosted.sh b/test/tag_for_hosted.sh
@@ -112,12 +112,12 @@ docker tag gcr.io/cloudsql-docker/gce-proxy:1.14 gcr.io/$PROJECT_ID/hosted/$COMM
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$SEM_VER
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$MM_VER
 
-docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
-docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
+docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
+docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
 
-docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
-docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
+docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
+docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
 docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
diff --git a/third_party/argo/Dockerfile.argoexec b/third_party/argo/Dockerfile.argoexec
@@ -23,7 +23,7 @@ RUN sh /third_party/download_source.sh </third_party/argo/repo-MPL.txt
 
 
 # Argo exec image
-From argoproj/argoexec:v2.3.0
+From argoproj/argoexec:v2.7.5
 
 # Copy concatenated license file
 COPY third_party/argo/license.txt /third_party/license.txt

diff --git a/third_party/argo/Dockerfile.workflow-controller b/third_party/argo/Dockerfile.workflow-controller
@@ -21,7 +21,7 @@ RUN apk add --no-cache curl
 ENV OUTPUT_FOLDER /downloads/
 RUN sh /third_party/download_source.sh </third_party/argo/repo-MPL.txt
 
-From argoproj/workflow-controller:v2.3.0
+From argoproj/workflow-controller:v2.7.5
 
 # Copy concatenated license file
 COPY third_party/argo/license.txt /third_party/license.txt

diff --git a/third_party/argo/release.sh b/third_party/argo/release.sh
@@ -20,7 +20,7 @@ set -e
 # $PROJECT: gcp project
 
 RELEASE_PROJECT=ml-pipeline
-TAG=v2.3.0-license-compliance
+TAG=v2.7.5-license-compliance
 
 gcloud builds submit --config third_party/argo/cloudbuild.yaml . \
   --substitutions=TAG_NAME="$TAG" --project "$PROJECT"