[Release] Automate release script for all the changes (#3777)

* Refactor components/release.sh to provide a new components/release-branch.sh that updates release branch directly

* Release components as version tag instead of commit SHA

* Publish component images in release.cloudbuild.yaml

* Include script that updates version tag for component sdk

* [Manifest] Use kustomize native image transformer to override image

* Revert unintended changes

* Fix kustomization.yaml location

* Fix inverse proxy image

* Add release script for kustomize manifest

* Add release scripts for marketplace manifest and sdk

* Add global release.sh

* Fix sdk release script

* Clean up release scripts

* Fix release script

* Fix release scripts

* fix

* fix

* Fix cannot use uppercase vars in cloudbuild.yaml

* Add old components release script back

* Add a RELEASE.md doc

.gitignore

@@ -66,3 +66,6 @@ _artifacts
 # Generated Python SDK documentation
 docs/_build
 
+# sed backups
+*.bak
+

.release.cloudbuild.yaml

@@ -24,6 +24,38 @@ steps:
       # Parse major minor version and save to a file for reusing in other steps.
       echo $TAG_NAME | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver
 
+# Pull and retag images for pipeline components
+- id: 'retagComponentImages'
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: bash
+  waitFor: ['-']
+  args:
+  - -ceux
+  - |
+    images=(
+      "ml-pipeline-kubeflow-deployer"
+      "ml-pipeline-kubeflow-tf-trainer"
+      "ml-pipeline-kubeflow-tf-trainer-gpu"
+      "ml-pipeline-kubeflow-tfjob"
+      "ml-pipeline-dataproc-analyze"
+      "ml-pipeline-dataproc-create-cluster"
+      "ml-pipeline-dataproc-delete-cluster"
+      "ml-pipeline-dataproc-predict"
+      "ml-pipeline-dataproc-transform"
+      "ml-pipeline-dataproc-train"
+      "ml-pipeline-local-confusion-matrix"
+      "ml-pipeline-local-roc"
+      "ml-pipeline-gcp"
+    )
+    for image in "${images[@]}"
+    do
+      from_image="gcr.io/$PROJECT_ID/$image:$COMMIT_SHA"
+      target_image="gcr.io/ml-pipeline/$image:$TAG_NAME"
+      docker pull $from_image
+      docker tag $from_image $target_image
+      docker push $target_image
+    done
+
 # Pull and retag the images for the pipeline system
 - name: 'gcr.io/cloud-builders/docker'
   args: ['pull', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA']

RELEASE.md

@@ -0,0 +1,51 @@
+# Releasing Kubeflow Pipelines
+
+WIP: this document is still incomplete.
+
+## Common Prerequisites
+
+* Permissions needed
+    * Can create a branch in github.com/kubeflow/pipelines
+* Tools that should be in your `$PATH`
+    * jq 1.6 https://stedolan.github.io/jq/download/
+    * yq https://github.com/mikefarah/yq/releases/tag/3.3.0
+    * jdk 8
+* Preparations
+    1. Clone github.com/kubeflow/pipelines repo into `$KFP_REPO`
+    2. `cd $KFP_REPO`
+
+## Cutting a release branch
+
+1. Choose a good commit on master branch with commit hash as `$COMMIT_SHA`
+2. Choose the next release branch's `$MINOR_VERSION` in format `x.y`, e.g. `1.0`, `1.1`...
+2. Make a release branch of format `release-$MINOR_VERSION`, e.g. `release-1.0`, `release-1.1`. Branch from the commit and push to kubeflow pipelines upstream repo.
+    ```bash
+    git checkout $COMMIT_SHA
+    git checkout -b release-$MINOR_VERSION
+    git push upstream HEAD
+    ```
+
+## Releasing from release branch
+
+1. Choose the release's complete `$VERSION` following semantic versioning, e.g.
+    * `1.0.0-rc.1`
+    * `1.0.0-rc.2`
+    * `1.0.0`
+    * `1.0.1`
+    * ...
+1. Update all version refs in release branch by
+    ```bash
+    ./hack/release.sh $VERSION release-$MINOR_VERSION
+    ```
+    It will prompt you whether to push it to release branch. Press `y` and hit `Enter`.
+
+    Note, the script will clone kubeflow/pipelines repo into a temporary location on your computer, make those changes and attempt to push to upstream, so that it won't interfere with your current git repo.
+
+    TODO: this script should also regenerate
+    * changelog
+    * python api client
+1. Wait and make sure the cloudbuild job that builds all images in gcr.io/ml-pipeline-test succeeded for above commit. Then submit the second cloudbuild job that copies these images to gcr.io/ml-pipeline.
+
+    TODO: we should have an automation KFP cluster, and the waiting and submiting second cloudbuild task should be automated.
+1. Release `kfp-server-api` and `kfp` python packages on pypi.
+1. Create a github release using `$VERSION` git tag, fill in the description.

components/release-in-place.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script automated the process to update the component images.
+# To run it, find a good release candidate commit SHA from ml-pipeline-test project,
+# and provide a full github COMMIT SHA to the script. E.g.
+# ./update-for-release.sh 2118baf752d3d30a8e43141165e13573b20d85b8
+# The script copies the images from test to prod, and update the local code.
+
+set -xe
+
+images=(
+  "ml-pipeline-kubeflow-deployer"
+  "ml-pipeline-kubeflow-tf-trainer"
+  "ml-pipeline-kubeflow-tf-trainer-gpu"
+  "ml-pipeline-kubeflow-tfjob"
+  "ml-pipeline-dataproc-analyze"
+  "ml-pipeline-dataproc-create-cluster"
+  "ml-pipeline-dataproc-delete-cluster"
+  "ml-pipeline-dataproc-predict"
+  "ml-pipeline-dataproc-transform"
+  "ml-pipeline-dataproc-train"
+  "ml-pipeline-local-confusion-matrix"
+  "ml-pipeline-local-roc"
+  "ml-pipeline-gcp"
+)
+
+TAG_NAME=$1
+FROM_GCR_PREFIX='gcr.io/ml-pipeline-test/'
+TO_GCR_PREFIX='gcr.io/ml-pipeline/'
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if [ -z "$TAG_NAME" ]; then
+  echo "Usage: release.sh <tag-name>" >&2
+  exit 1
+fi
+
+# KFP repo root
+pushd "$DIR/.."
+
+# Update setup.py VERSION
+sed -i.bak -e "s|VERSION =.\+'|VERSION = '${TAG_NAME}'|g" "components/gcp/container/component_sdk/python/setup.py"
+
+# Updating components and samples.
+for image in "${images[@]}"
+do
+  TARGET_IMAGE_BASE=${TO_GCR_PREFIX}${image}
+  TARGET_IMAGE=${TARGET_IMAGE_BASE}:${TAG_NAME}
+
+  # Update the code
+  find components samples -type f | while read file; do
+    sed -i -e "s|${TARGET_IMAGE_BASE}:\([a-zA-Z0-9_.-]\)\+|${TARGET_IMAGE}|g" "$file"
+  done
+done
+
+# Updating the samples to use the updated components
+git diff --name-only | while read component_file; do
+    echo $component_file
+    find components samples -type f | while read file; do
+      sed -i -E "s|(https://raw.githubusercontent.com/kubeflow/pipelines/)[^/]+(/$component_file)|\1${TAG_NAME}\2|g" "$file";
+    done
+done
+popd

hack/release-imp.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+TAG_NAME=$1
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if [[ -z "$TAG_NAME" ]]; then
+  echo "Usage: release.sh <release-tag>" >&2
+  exit 1
+fi
+
+"$DIR/../components/release-in-place.sh" $TAG_NAME
+"$DIR/../manifests/gcp_marketplace/hack/release.sh" $TAG_NAME
+"$DIR/../manifests/kustomize/hack/release.sh" $TAG_NAME
+"$DIR/../sdk/hack/release.sh" $TAG_NAME
+echo "$TAG_NAME" > "$DIR/../VERSION"

hack/release.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+TAG_NAME=$1
+BRANCH=$2
+REPO=kubeflow/pipelines
+
+if [[ -z "$BRANCH" || -z "$TAG_NAME" ]]; then
+  echo "Usage: release.sh <release-tag> <release-branch>" >&2
+  exit 1
+fi
+
+# Checking out the repo's release branch
+clone_dir=$(mktemp -d)
+git clone "git@github.com:${REPO}.git" "$clone_dir"
+cd "$clone_dir"
+git checkout "$BRANCH"
+
+# Run the release script in cloned repo
+"hack/release-imp.sh" $TAG_NAME
+
+# Checking-in the component changes
+git add --all
+git commit --message "Updated version to $TAG_NAME"
+git tag -a "$TAG_NAME" -m "Kubeflow Pipelines $TAG_NAME release"
+
+# Pushing the changes upstream
+read -p "Do you want to push the version change and tag $TAG_NAME tag to upstream? [y|n]"
+if [ "$REPLY" != "y" ]; then
+   exit
+fi
+git push --set-upstream origin "$BRANCH"
+git push origin "$TAG_NAME"

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml

@@ -28,7 +28,7 @@ spec:
 
   info:
     - name: Application Namespace
-      value: {{ .Release.Namespace }}
+      value: "{{ .Release.Namespace }}"
     - name: Console
       value: 'https://console.cloud.google.com/ai-platform/pipelines/clusters'
   componentKinds:

manifests/gcp_marketplace/hack/release.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+TAG_NAME=$1
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if [[ -z "$TAG_NAME" ]]; then
+  echo "Usage: release.sh <release-tag>" >&2
+  exit 1
+fi
+
+echo "This release script uses yq, it can be downloaded at https://github.com/mikefarah/yq/releases/tag/3.3.0"
+yq w -i "$DIR/../schema.yaml" "x-google-marketplace.publishedVersion" "$TAG_NAME"
+yq w -i "$DIR/../chart/kubeflow-pipelines/templates/application.yaml" "spec.descriptor.version" "$TAG_NAME"

manifests/kustomize/hack/release.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+TAG_NAME=$1
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if [[ -z "$TAG_NAME" ]]; then
+  echo "Usage: release.sh <release-tag>" >&2
+  exit 1
+fi
+
+echo "This release script uses yq, it can be downloaded at https://github.com/mikefarah/yq/releases/tag/3.3.0"
+yq w -i "$DIR/../base/kustomization.yaml" images[*].newTag "$TAG_NAME"
+yq w -i "$DIR/../env/gcp/inverse-proxy/kustomization.yaml" images[*].newTag "$TAG_NAME"

sdk/hack/release.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+TAG_NAME=$1
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+if [[ -z "$TAG_NAME" ]]; then
+  echo "Usage: release.sh <release-tag>" >&2
+  exit 1
+fi
+
+sed -i.bak -e "s|__version__ =.\+|__version__ = '$TAG_NAME'|g" "$DIR/../python/kfp/__init__.py"