diff --git a/.gitignore b/.gitignore index b1d84f8597a..701b5982f5a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ _artifacts # Generated Python SDK documentation docs/_build +# sed backups +*.bak + diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 886109c2028..d6bf8946c97 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -24,6 +24,38 @@ steps: # Parse major minor version and save to a file for reusing in other steps. echo $TAG_NAME | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver +# Pull and retag images for pipeline components +- id: 'retagComponentImages' + name: 'gcr.io/cloud-builders/docker' + entrypoint: bash + waitFor: ['-'] + args: + - -ceux + - | + images=( + "ml-pipeline-kubeflow-deployer" + "ml-pipeline-kubeflow-tf-trainer" + "ml-pipeline-kubeflow-tf-trainer-gpu" + "ml-pipeline-kubeflow-tfjob" + "ml-pipeline-dataproc-analyze" + "ml-pipeline-dataproc-create-cluster" + "ml-pipeline-dataproc-delete-cluster" + "ml-pipeline-dataproc-predict" + "ml-pipeline-dataproc-transform" + "ml-pipeline-dataproc-train" + "ml-pipeline-local-confusion-matrix" + "ml-pipeline-local-roc" + "ml-pipeline-gcp" + ) + for image in "${images[@]}" + do + from_image="gcr.io/$PROJECT_ID/$image:$COMMIT_SHA" + target_image="gcr.io/ml-pipeline/$image:$TAG_NAME" + docker pull $from_image + docker tag $from_image $target_image + docker push $target_image + done + # Pull and retag the images for the pipeline system - name: 'gcr.io/cloud-builders/docker' args: ['pull', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'] diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000000..6de2eed39a4 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,51 @@ +# Releasing Kubeflow Pipelines + +WIP: this document is still incomplete. + +## Common Prerequisites + +* Permissions needed + * Can create a branch in github.com/kubeflow/pipelines +* Tools that should be in your `$PATH` + * jq 1.6 https://stedolan.github.io/jq/download/ + * yq https://github.com/mikefarah/yq/releases/tag/3.3.0 + * jdk 8 +* Preparations + 1. Clone github.com/kubeflow/pipelines repo into `$KFP_REPO` + 2. `cd $KFP_REPO` + +## Cutting a release branch + +1. Choose a good commit on master branch with commit hash as `$COMMIT_SHA` +2. Choose the next release branch's `$MINOR_VERSION` in format `x.y`, e.g. `1.0`, `1.1`... +2. Make a release branch of format `release-$MINOR_VERSION`, e.g. `release-1.0`, `release-1.1`. Branch from the commit and push to kubeflow pipelines upstream repo. + ```bash + git checkout $COMMIT_SHA + git checkout -b release-$MINOR_VERSION + git push upstream HEAD + ``` + +## Releasing from release branch + +1. Choose the release's complete `$VERSION` following semantic versioning, e.g. + * `1.0.0-rc.1` + * `1.0.0-rc.2` + * `1.0.0` + * `1.0.1` + * ... +1. Update all version refs in release branch by + ```bash + ./hack/release.sh $VERSION release-$MINOR_VERSION + ``` + It will prompt you whether to push it to release branch. Press `y` and hit `Enter`. + + Note, the script will clone kubeflow/pipelines repo into a temporary location on your computer, make those changes and attempt to push to upstream, so that it won't interfere with your current git repo. + + TODO: this script should also regenerate + * changelog + * python api client +1. Wait and make sure the cloudbuild job that builds all images in gcr.io/ml-pipeline-test succeeded for above commit. Then submit the second cloudbuild job that copies these images to gcr.io/ml-pipeline. + + TODO: we should have an automation KFP cluster, and the waiting and submiting second cloudbuild task should be automated. +1. Release `kfp-server-api` and `kfp` python packages on pypi. +1. Create a github release using `$VERSION` git tag, fill in the description. diff --git a/components/release-in-place.sh b/components/release-in-place.sh new file mode 100755 index 00000000000..bbd1fe9f6df --- /dev/null +++ b/components/release-in-place.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script automated the process to update the component images. +# To run it, find a good release candidate commit SHA from ml-pipeline-test project, +# and provide a full github COMMIT SHA to the script. E.g. +# ./update-for-release.sh 2118baf752d3d30a8e43141165e13573b20d85b8 +# The script copies the images from test to prod, and update the local code. + +set -xe + +images=( + "ml-pipeline-kubeflow-deployer" + "ml-pipeline-kubeflow-tf-trainer" + "ml-pipeline-kubeflow-tf-trainer-gpu" + "ml-pipeline-kubeflow-tfjob" + "ml-pipeline-dataproc-analyze" + "ml-pipeline-dataproc-create-cluster" + "ml-pipeline-dataproc-delete-cluster" + "ml-pipeline-dataproc-predict" + "ml-pipeline-dataproc-transform" + "ml-pipeline-dataproc-train" + "ml-pipeline-local-confusion-matrix" + "ml-pipeline-local-roc" + "ml-pipeline-gcp" +) + +TAG_NAME=$1 +FROM_GCR_PREFIX='gcr.io/ml-pipeline-test/' +TO_GCR_PREFIX='gcr.io/ml-pipeline/' +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" + +if [ -z "$TAG_NAME" ]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +# KFP repo root +pushd "$DIR/.." + +# Update setup.py VERSION +sed -i.bak -e "s|VERSION =.\+'|VERSION = '${TAG_NAME}'|g" "components/gcp/container/component_sdk/python/setup.py" + +# Updating components and samples. +for image in "${images[@]}" +do + TARGET_IMAGE_BASE=${TO_GCR_PREFIX}${image} + TARGET_IMAGE=${TARGET_IMAGE_BASE}:${TAG_NAME} + + # Update the code + find components samples -type f | while read file; do + sed -i -e "s|${TARGET_IMAGE_BASE}:\([a-zA-Z0-9_.-]\)\+|${TARGET_IMAGE}|g" "$file" + done +done + +# Updating the samples to use the updated components +git diff --name-only | while read component_file; do + echo $component_file + find components samples -type f | while read file; do + sed -i -E "s|(https://raw.githubusercontent.com/kubeflow/pipelines/)[^/]+(/$component_file)|\1${TAG_NAME}\2|g" "$file"; + done +done +popd diff --git a/components/release.sh b/components/release.sh old mode 100755 new mode 100644 diff --git a/hack/release-imp.sh b/hack/release-imp.sh new file mode 100755 index 00000000000..d8df219c61c --- /dev/null +++ b/hack/release-imp.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +TAG_NAME=$1 +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" + +if [[ -z "$TAG_NAME" ]]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +"$DIR/../components/release-in-place.sh" $TAG_NAME +"$DIR/../manifests/gcp_marketplace/hack/release.sh" $TAG_NAME +"$DIR/../manifests/kustomize/hack/release.sh" $TAG_NAME +"$DIR/../sdk/hack/release.sh" $TAG_NAME +echo "$TAG_NAME" > "$DIR/../VERSION" diff --git a/hack/release.sh b/hack/release.sh new file mode 100755 index 00000000000..d2d0ba9e791 --- /dev/null +++ b/hack/release.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +TAG_NAME=$1 +BRANCH=$2 +REPO=kubeflow/pipelines + +if [[ -z "$BRANCH" || -z "$TAG_NAME" ]]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +# Checking out the repo's release branch +clone_dir=$(mktemp -d) +git clone "git@github.com:${REPO}.git" "$clone_dir" +cd "$clone_dir" +git checkout "$BRANCH" + +# Run the release script in cloned repo +"hack/release-imp.sh" $TAG_NAME + +# Checking-in the component changes +git add --all +git commit --message "Updated version to $TAG_NAME" +git tag -a "$TAG_NAME" -m "Kubeflow Pipelines $TAG_NAME release" + +# Pushing the changes upstream +read -p "Do you want to push the version change and tag $TAG_NAME tag to upstream? [y|n]" +if [ "$REPLY" != "y" ]; then + exit +fi +git push --set-upstream origin "$BRANCH" +git push origin "$TAG_NAME" diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml index 4a9d3e71929..326bf8fb98c 100644 --- a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml +++ b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml @@ -28,7 +28,7 @@ spec: info: - name: Application Namespace - value: {{ .Release.Namespace }} + value: "{{ .Release.Namespace }}" - name: Console value: 'https://console.cloud.google.com/ai-platform/pipelines/clusters' componentKinds: diff --git a/manifests/gcp_marketplace/hack/release.sh b/manifests/gcp_marketplace/hack/release.sh new file mode 100755 index 00000000000..a514b5f6285 --- /dev/null +++ b/manifests/gcp_marketplace/hack/release.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +TAG_NAME=$1 +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" + +if [[ -z "$TAG_NAME" ]]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +echo "This release script uses yq, it can be downloaded at https://github.com/mikefarah/yq/releases/tag/3.3.0" +yq w -i "$DIR/../schema.yaml" "x-google-marketplace.publishedVersion" "$TAG_NAME" +yq w -i "$DIR/../chart/kubeflow-pipelines/templates/application.yaml" "spec.descriptor.version" "$TAG_NAME" diff --git a/manifests/kustomize/hack/release.sh b/manifests/kustomize/hack/release.sh new file mode 100755 index 00000000000..8f3cf7aaec7 --- /dev/null +++ b/manifests/kustomize/hack/release.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +TAG_NAME=$1 +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" + +if [[ -z "$TAG_NAME" ]]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +echo "This release script uses yq, it can be downloaded at https://github.com/mikefarah/yq/releases/tag/3.3.0" +yq w -i "$DIR/../base/kustomization.yaml" images[*].newTag "$TAG_NAME" +yq w -i "$DIR/../env/gcp/inverse-proxy/kustomization.yaml" images[*].newTag "$TAG_NAME" diff --git a/sdk/hack/release.sh b/sdk/hack/release.sh new file mode 100755 index 00000000000..f72793fbaef --- /dev/null +++ b/sdk/hack/release.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +TAG_NAME=$1 +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" + +if [[ -z "$TAG_NAME" ]]; then + echo "Usage: release.sh " >&2 + exit 1 +fi + +sed -i.bak -e "s|__version__ =.\+|__version__ = '$TAG_NAME'|g" "$DIR/../python/kfp/__init__.py"