HBASE-24318 Create-release scripts fixes and enhancements (#1643)

* narrow 'dry-run' limits so see svn activity up to just before check-in.
* Fix several typos and, in case of dry run, enable build step to chain from tag step by keeping the tagged git repo.
* Improve Maven settings.xml file, and names of variables related to it. Remove unnecessary use of "-Dmaven.repo.local", put it in settings.xml instead. Stop putting password literals in settings.xml. 
* stop deleting maven settings file, now that it doesn't contain password strings
* Merge 'tag' into release-build.sh as another stage. Also found and fixed a couple bugs in the current release-tag.sh.
* Delete redundant release-tag.sh script.
* Small changes to make dev-support/create-release tools less focused
on hbase project only, while retaining special behaviors for
hbase sub-projects which share the hbase dist and jira locations.
* Changed terminology of release publish steps, from <build|publish>
to <publish-dist|publish-release>. In fact, what was formerly called
"build" built the distribution tarballs AND published them to dist,
while "publish" built the  maven artifacts AND published them to Nexus.
The new terminology clarifies what's happening, and removes the appearance
of order dependendency.
* Fix publish-snapshot so it does same checks as publish-release.
* Factor out common maven usages, and move them to build-util.sh.
* Change default polarity of DRY_RUN to default to true. Change -n ("no-publish") to -f ("force publish") to actually publish.
* Fix problems in do-release.sh so it runs correctly outside of docker,
  including DRY_RUN being exported.
* Have do-release.sh set REPO (shared maven local repository) if doing all three stages.
* Cleaned up REPO directory creation.
* General cleanup of comments and usage.
*  fix all 'shellcheck' errors
* use ${BASH_SOURCE[0]} instead of $0 to determine script directory path
* smarter way to read version from pom with mvn
* do maven-gpg-plugin config settings in maven settings file correctly as documented
* fix gpg signing failure on Mac due to gpg-agent timeout
* fix various bugs to enable publish-dist, publish-snapshot, and publish-release to work correctly as individual steps and/or without docker
* improve log reporting from publish-release step
*  fix bug in argument to checkcompatibility.py: replace PACKAGE_VERSION with GIT_REF
* demote "PACKAGE_VERSION" to "package_version_name" and undocument it in favor of RELEASE_TAG. Still enable appropriate defaulting in case RELEASE_TAG is undefined.
* unify RELEASE_VERSION with VERSION, to remove ambiguity and allow it to be set when only running 'publish' step without 'tag'
* query confirm RELEASE_TAG
* emphasize that release-build.sh is called for a single action at a time, and should be called from do-release.sh.
* add '-s' option to do-release.sh
* suppress maven "Download from central:" messages
* Replace human wait for tag propagation

Signed-off-by: Cesar Delgado <cdelgado@apple.com>
Signed-off-by: stack <stack@apache.org>
Signed-off-by: Sean Busbey <busbey@apache.org>

Author: mattf-apache

.gitignore

@@ -21,3 +21,5 @@ linklint/
 .checkstyle
 **/.checkstyle
 .java-version
+*.log
+**/*.log

dev-support/create-release/README.txt

@@ -5,9 +5,11 @@ For usage, pass '-h':
 
  $ ./do-release-docker.sh -h
 
-To run a build w/o invoking docker (not recommeneded!), use
-_do_release.sh_. It does not take parameters. It will ask
-you what commands to run with taking defaults from environment.
+To run a build w/o invoking docker (not recommended!), use _do_release.sh_.
+
+Both scripts will query interactively for needed parameters and passphrases.
+For explanation of the parameters, execute:
+ $ release-build.sh --help
 
 Before starting the RC build, run a reconciliation of what is in
 JIRA with what is in the commit log. Make sure they align and that
@@ -19,7 +21,6 @@ Running a build on GCE is easy enough. Here are some notes if of use.
 Create an instance. 4CPU/15G/10G disk seems to work well enough.
 Once up, run the below to make your machine fit for RC building:
 
-
 # Presuming debian-compatible OS
 $ sudo apt-get install -y git openjdk-8-jdk maven gnupg gnupg-agent
 # Install docker
@@ -37,13 +38,14 @@ $ sudo add-apt-repository -y \
 $ sudo apt-get update
 $ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
 $ sudo usermod -a -G docker $USERID
-# LOGOUT and then LOGIN again so $USERID shows as part of docker groupl
+# LOGOUT and then LOGIN again so $USERID shows as part of docker group
 # Copy up private key for $USERID export from laptop and import on gce.
 $ gpg --import stack.duboce.net.asc
 $ export GPG_TTY=$(tty) # https://github.com/keybase/keybase-issues/issues/2798
 $ eval $(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
-$ git clone https://github.com/apache/hbase.git
-$ cd hbase
+$ export PROJECT="${PROJECT:-hbase}"
+$ git clone https://github.com/apache/${PROJECT}.git
+$ cd "${PROJECT}"
 $ mkdir ~/build
 $ ./dev-resources/create-release/do-release-docker.sh -d ~/build
 # etc.

dev-support/create-release/do-release-docker.sh

@@ -47,15 +47,16 @@
 #
 set -e
 
-# Set this building other hbase repos: e.g. PROJECT=hbase-operator-tools
+# Set this to build other hbase repos: e.g. PROJECT=hbase-operator-tools
 export PROJECT="${PROJECT:-hbase}"
 
-SELF=$(cd $(dirname "$0") && pwd)
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
 
 function usage {
   local NAME
-  NAME="$(basename "$0")"
+  NAME="$(basename "${BASH_SOURCE[0]}")"
   cat <<EOF
 Usage: $NAME [options]
 
@@ -64,24 +65,26 @@ This script runs the release scripts inside a docker image.
 Options:
 
   -d [path]    required. working directory. output will be written to "output" in here.
-  -n           dry run mode. Checks and local builds, but does not upload anything.
+  -f           "force" -- actually publish this release. Unless you specify '-f', it will
+               default to dry run mode, which checks and does local builds, but does not upload anything.
   -t [tag]     tag for the hbase-rm docker image to use for building (default: "latest").
   -j [path]    path to local JDK installation to use building. By default the script will
                use openjdk8 installed in the docker image.
-  -p [project] project to build; default 'hbase'; alternatively, 'hbase-thirdparty', etc.
-  -s [step]    runs a single step of the process; valid steps are: tag, build, publish. if
-               none specified, runs tag, then build, and then publish.
+  -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+  -s [step]    runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
+               If none specified, runs tag, then publish-dist, and then publish-release.
+               'publish-snapshot' is also an allowed, less used, option.
 EOF
 }
 
 WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:hj:np:s:t:" opt; do
+while getopts "d:fhj:p:s:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
-    n) DRY_RUN=1 ;;
+    f) DRY_RUN=0 ;;
     t) IMGTAG="$OPTARG" ;;
     j) JAVA="$OPTARG" ;;
     p) PROJECT="$OPTARG" ;;
@@ -90,6 +93,10 @@ while getopts "d:hj:np:s:t:" opt; do
     ?) error "Invalid option. Run with -h for help." ;;
   esac
 done
+shift $((OPTIND-1))
+if (( $# > 0 )); then
+  error "Arguments can only be provided with option flags, invalid args: $*"
+fi
 
 if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
   error "Work directory (-d) must be defined and exist. Run with -h for help."
@@ -145,27 +152,27 @@ NEXT_VERSION=$NEXT_VERSION
 RELEASE_VERSION=$RELEASE_VERSION
 RELEASE_TAG=$RELEASE_TAG
 GIT_REF=$GIT_REF
-PACKAGE_VERSION=$PACKAGE_VERSION
 ASF_USERNAME=$ASF_USERNAME
 GIT_NAME=$GIT_NAME
 GIT_EMAIL=$GIT_EMAIL
 GPG_KEY=$GPG_KEY
 ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
 RELEASE_STEP=$RELEASE_STEP
-RELEASE_STEP=$RELEASE_STEP
 API_DIFF_TAG=$API_DIFF_TAG
 EOF
 
-JAVA_VOL=
+JAVA_VOL=()
 if [ -n "$JAVA" ]; then
   echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
-  JAVA_VOL="--volume $JAVA:/opt/hbase-java"
+  JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
 fi
 
 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
-docker run -ti \
+cmd=(docker run -ti \
   --env-file "$ENVFILE" \
   --volume "$WORKDIR:/opt/hbase-rm" \
-  $JAVA_VOL \
-  "hbase-rm:$IMGTAG"
+  "${JAVA_VOL[@]}" \
+  "hbase-rm:$IMGTAG")
+echo "${cmd[*]}"
+"${cmd[@]}"

dev-support/create-release/do-release.sh

@@ -23,24 +23,32 @@
 # and passwords to use building.
 export PROJECT="${PROJECT:-hbase}"
 
-SELF=$(cd $(dirname $0) && pwd)
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
 
-while getopts "bn" opt; do
+while getopts "b:fs:" opt; do
   case $opt in
-    b) GIT_BRANCH=$OPTARG ;;
-    n) DRY_RUN=1 ;;
+    b) export GIT_BRANCH=$OPTARG ;;
+    f) export DRY_RUN=0 ;;  # "force", ie actually publish this release (otherwise defaults to dry run)
+    s) RELEASE_STEP="$OPTARG" ;;
     ?) error "Invalid option: $OPTARG" ;;
   esac
 done
+shift $((OPTIND-1))
+if (( $# > 0 )); then
+  error "Arguments can only be provided with option flags, invalid args: $*"
+fi
 
 # If running in docker, import and then cache keys.
 if [ "$RUNNING_IN_DOCKER" = "1" ]; then
   # Run gpg agent.
-  eval $(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
-  echo "GPG Version: `gpg --version`"
-  # Inside docker, need to import the GPG key stored in the current directory.
-  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
+  eval "$(gpg-agent --disable-scdaemon --daemon --no-grab  --allow-preset-passphrase \
+          --default-cache-ttl=86400 --max-cache-ttl=86400)"
+  echo "GPG Version: $(gpg --version)"
+  # Inside docker, need to import the GPG keyfile stored in the current directory.
+  # (On workstation, assume GPG has access to keychain/cache with key_id already imported.)
+  echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
 
   # We may need to adjust the path since JAVA_HOME may be overridden by the driver script.
   if [ -n "$JAVA_HOME" ]; then
@@ -53,7 +61,15 @@ else
   # Outside docker, need to ask for information about the release.
   get_release_info
 fi
-export GPG_TTY=$(tty)
+GPG_TTY="$(tty)"
+export GPG_TTY
+
+if [[ -z "$RELEASE_STEP" ]]; then
+  # If doing all stages, leave out 'publish-snapshot'
+  RELEASE_STEP="tag_publish-dist_publish-release"
+  # and use shared maven local repo for efficiency
+  export REPO="${REPO:-$(pwd)/$(mktemp -d hbase-repo-XXXXX)}"
+fi
 
 function should_build {
   local WHAT=$1
@@ -66,26 +82,30 @@ function should_build {
   fi
 }
 
-if should_build "tag" && [ $SKIP_TAG = 0 ]; then
+if should_build "tag" && [ "$SKIP_TAG" = 0 ]; then
   run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
-    "$SELF/release-tag.sh"
-  echo "It may take some time for the tag to be synchronized to github."
-  echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available."
-  read
+    "$SELF/release-build.sh" tag
+  if is_dry_run; then
+    export TAG_SAME_DRY_RUN="true";
+  fi
 else
   echo "Skipping tag creation for $RELEASE_TAG."
 fi
 
-if should_build "build"; then
-  run_silent "Building ${PROJECT}..." "build.log" \
-    "$SELF/release-build.sh" build
+if should_build "publish-dist"; then
+  run_silent "Publishing distribution packages (tarballs)" "publish-dist.log" \
+    "$SELF/release-build.sh" publish-dist
 else
-  echo "Skipping build step."
+  echo "Skipping publish-dist step."
 fi
 
-if should_build "publish"; then
-  run_silent "Publishing release" "publish.log" \
+if should_build "publish-snapshot"; then
+  run_silent "Publishing snapshot" "publish-snapshot.log" \
+    "$SELF/release-build.sh" publish-snapshot
+
+elif should_build "publish-release"; then
+  run_silent "Publishing release" "publish-release.log" \
     "$SELF/release-build.sh" publish-release
 else
-  echo "Skipping publish step."
+  echo "Skipping publish-release step."
 fi