[Release] Regenerate api clients on release (#3910)

* [Release] Regenerate api clients on release

* Fix hack/release.sh

* Add a script to check required tools

* update

* Update

* Update clean up

backend/api/build_kfp_server_api_python_package.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Copyright 2018 Google LLC
+# Copyright 2018-2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,10 +27,11 @@
 # brew cask install caskroom/versions/java8
 # brew install jq
 
-VERSION="$1"
-
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+REPO_ROOT="$DIR/../.."
+VERSION="$(cat $REPO_ROOT/VERSION)"
 if [ -z "$VERSION" ]; then
-    echo "Usage: build_kfp_server_api_python_package.sh <version>"
+    echo "ERROR: $REPO_ROOT/VERSION is empty"
     exit 1
 fi
 

backend/api/generate_api.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2018 Google LLC
+# Copyright 2018-2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,9 +21,15 @@
 
 set -ex
 
-VERSION="1.0.0-dev.1"
-BAZEL_BINDIR=$(bazel info bazel-bin)
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+REPO_ROOT="$DIR/../.."
+VERSION="$(cat $REPO_ROOT/VERSION)"
+if [ -z "$VERSION" ]; then
+    echo "ERROR: $REPO_ROOT/VERSION is empty"
+    exit 1
+fi
+
+BAZEL_BINDIR=$(bazel info bazel-bin)
 SWAGGER_CMD=${DIR}/../../bazel-bin/external/com_github_go_swagger/cmd/swagger/*stripped/swagger
 AUTOGEN_CMD="${DIR}/../../bazel-bin/external/com_github_mbrukman_autogen/autogen_tool"
 GENERATED_GO_PROTO_FILES="${BAZEL_BINDIR}/backend/api/api_generated_go_sources/src/github.com/kubeflow/pipelines/backend/api/go_client/*.go"
@@ -41,7 +47,6 @@ bazel build @com_github_go_swagger//cmd/swagger
 # Build .pb.go and .gw.pb.go files from the proto sources.
 bazel build //backend/api:api_generated_go_sources
 
-set -x
 # Copy the generated files into the source tree and add license.
 for f in $GENERATED_GO_PROTO_FILES; do
   target=${DIR}/go_client/$(basename ${f})
@@ -127,3 +132,5 @@ find ${DIR}/go_http_client/ -name "*.go" -exec ${AUTOGEN_CMD} -i --no-tlc -c "Go
 
 # Finally, run gazelle to add BUILD files for the generated code.
 bazel run //:gazelle
+# HACK: remove unnecessary BUILD.bazels
+rm -f "$REPO_ROOT/sdk/python/kfp/components/structures/BUILD.bazel" "$REPO_ROOT/tools/metadatastore-upgrade/BUILD.bazel"

hack/check-release-needed-tools.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+echo "The following tools are needed when releasing KFP:"
+echo "jq>=1.6"
+which jq || (echo "jq not found in PATH" && exit 1)
+echo "yq>=3.3"
+which yq || (echo "yq not found in PATH" && exit 1)
+yq -V | grep 3. || (echo "yq version 3.x should be used" && exit 1)
+echo "java>=8"
+which java || (echo "java not found in PATH" && exit 1)
+echo "bazel==0.24.0"
+which bazel || (echo "bazel not found in PATH" && exit 1)
+bazel version | grep 0.24.0 || (echo "bazel not 0.24.0 version" && exit 1)
+echo "python>3"
+which python || (echo "python not found in PATH" && exit 1)
+python -c "import setuptools" || (echo "setuptools should be installed in python" && exit 1)
+
+echo "All tools installed"
+echo "Please add another needed tools if above list is not complete"

hack/release-imp.sh

@@ -16,16 +16,22 @@
 
 set -ex
 
-TAG_NAME=$1
+echo "Usage: edit kubeflow/pipelines/VERSION to new version tag first, then run this script."
+
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+REPO_ROOT="$DIR/.."
+TAG_NAME="$(cat $REPO_ROOT/VERSION)"
 
 if [[ -z "$TAG_NAME" ]]; then
-  echo "Usage: release.sh <release-tag>" >&2
+  echo "ERROR: $REPO_ROOT/VERSION is empty" >&2
   exit 1
 fi
 
-"$DIR/../components/release-in-place.sh" $TAG_NAME
-"$DIR/../manifests/gcp_marketplace/hack/release.sh" $TAG_NAME
-"$DIR/../manifests/kustomize/hack/release.sh" $TAG_NAME
-"$DIR/../sdk/hack/release.sh" $TAG_NAME
-echo "$TAG_NAME" > "$DIR/../VERSION"
+"$DIR/check-release-needed-tools.sh"
+
+"$REPO_ROOT/components/release-in-place.sh" $TAG_NAME
+"$REPO_ROOT/manifests/gcp_marketplace/hack/release.sh" $TAG_NAME
+"$REPO_ROOT/manifests/kustomize/hack/release.sh" $TAG_NAME
+"$REPO_ROOT/sdk/hack/release.sh" $TAG_NAME
+"$REPO_ROOT/backend/api/generate_api.sh"
+"$REPO_ROOT/backend/api/build_kfp_server_api_python_package.sh"

hack/release.sh

@@ -31,6 +31,7 @@ git clone "git@github.com:${REPO}.git" "$clone_dir"
 cd "$clone_dir"
 git checkout "$BRANCH"
 
+echo "$TAG_NAME" > ./VERSION
 # Run the release script in cloned repo
 "hack/release-imp.sh" $TAG_NAME