Add release tools

dev/build-wheels.sh

@@ -19,7 +19,8 @@ set -e -x
 dev/lint-python.sh -s py_env
 
 PY_ENV_DIR=`pwd`/dev/.conda/envs
-py_env=("3.8" "3.9" "3.10" "3.11")
+# Don't need to build with different Python version when there are no C-related codes
+py_env=("3.11")
 ## 2. install dependency
 for ((i=0;i<${#py_env[@]};i++)) do
     echo "Installing dependencies for environment: ${py_env[i]}"

dev/lint-python.sh

@@ -571,7 +571,6 @@ function check_stage() {
 #########################
 # Tox check
 function tox_check() {
-    LATEST_PYTHON="py311"
     print_function "STAGE" "tox checks"
     # Set created py-env in $PATH for tox's creating virtual env
     activate
@@ -581,11 +580,11 @@ function tox_check() {
     # tox runs codes in virtual env, set var to avoid error
     export _PYPAIMON_TOX_TEST="true"
 
-    if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then
-        # Only run test in latest python version triggered by a Git push
-        $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini -e ${LATEST_PYTHON} --recreate 2>&1 | tee -a $LOG_FILE
+    if [[ -n "$GITHUB_ACTION" ]]; then
+        # Run tests in all versions triggered by a Git push (tests aren't so many currently)
+        $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE
     else
-        # Only run random selected python version in nightly CI.
+        # Only run random selected python version at local.
         ENV_LIST_STRING=`$TOX_PATH -l -c $PAIMON_PYTHON_DIR/tox.ini`
         _OLD_IFS=$IFS
         IFS=$'\n'

setup.py

@@ -20,6 +20,7 @@
 import os
 import shutil
 import setup_utils.java_setuputils as java_setuputils
+import setup_utils.version
 
 from setuptools import Command, setup
 
@@ -72,7 +73,7 @@ def run(self):
 
     setup(
         name='paimon_python',
-        version='0.1.0.dev0',
+        version=setup_utils.version.__version__,
         packages=PACKAGES,
         include_package_data=True,
         package_data=PACKAGE_DATA,

setup_utils/version.py

@@ -0,0 +1,22 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""
+.. seealso:: https://www.python.org/dev/peps/pep-0440
+"""
+__version__ = "0.1.dev0"

tools/releasing/create_binary_release.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Required variables
+##
+RELEASE_VERSION=${RELEASE_VERSION}
+
+if [ -z "${RELEASE_VERSION}" ]; then
+	echo "RELEASE_VERSION was not set"
+	exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+
+CURR_DIR=`pwd`
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+PROJECT_ROOT="${BASE_DIR}/../../"
+
+# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
+if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
+    echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
+    exit 1
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+    SHASUM="shasum -a 512"
+else
+    SHASUM="sha512sum"
+fi
+
+###########################
+
+RELEASE_DIR=${PROJECT_ROOT}/release/binary
+rm -rf ${RELEASE_DIR}
+mkdir -p ${RELEASE_DIR}
+
+# use lint-python.sh script to create a python environment.
+dev/lint-python.sh -s basic
+source dev/.conda/bin/activate
+
+# build
+dev/build-wheels.sh
+
+WHEEL_FILE_NAME="paimon_python-${RELEASE_VERSION}-py3-none-any.whl"
+WHEEL_FILE="${RELEASE_DIR}/${WHEEL_FILE_NAME}"
+cp "dist/${WHEEL_FILE_NAME}" ${WHEEL_FILE}
+
+# Sign sha the wheel package
+gpg --armor --detach-sig ${WHEEL_FILE}
+$SHASUM ${WHEEL_FILE} > "${WHEEL_FILE}.sha512"

tools/releasing/create_release_branch.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+RELEASE_CANDIDATE=${RELEASE_CANDIDATE:-none}
+
+##
+## Required variables
+##
+RELEASE_VERSION=${RELEASE_VERSION}
+
+if [ -z "${RELEASE_VERSION}" ]; then
+	echo "RELEASE_VERSION was not set"
+	exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+
+CURR_DIR=`pwd`
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+PROJECT_ROOT="${BASE_DIR}/../../"
+
+# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
+if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
+    echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
+    exit 1
+fi
+
+###########################
+
+TARGET_BRANCH=release-${RELEASE_VERSION}
+if [ "${RELEASE_CANDIDATE}" != "none" ]; then
+  TARGET_BRANCH=${TARGET_BRANCH}-rc${RELEASE_CANDIDATE}
+fi
+
+cd ${PROJECT_ROOT}
+git checkout -b ${TARGET_BRANCH}
+
+RELEASE_COMMIT_HASH=`git rev-parse HEAD`
+echo "Done. Created a new release branch with commit hash ${RELEASE_COMMIT_HASH}."
+
+cd ${CURR_DIR}

tools/releasing/create_source_release.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Required variables
+##
+RELEASE_VERSION=${RELEASE_VERSION}
+
+if [ -z "${RELEASE_VERSION}" ]; then
+	echo "RELEASE_VERSION is unset"
+	exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+
+CURR_DIR=`pwd`
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+PROJECT_ROOT="$( cd "$( dirname "${BASE_DIR}/../../../" )" >/dev/null && pwd )"
+
+# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
+if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
+    echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
+    exit 1
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+    SHASUM="shasum -a 512"
+    TAR="tar --no-xattrs"
+else
+    SHASUM="sha512sum"
+    TAR="tar"
+fi
+
+###########################
+
+RELEASE_DIR=${PROJECT_ROOT}/release/source
+CLONE_DIR=${RELEASE_DIR}/paimon-tmp-clone
+
+rm -rf ${RELEASE_DIR}
+mkdir -p ${RELEASE_DIR}
+
+# delete the temporary release directory on error
+trap 'rm -rf ${RELEASE_DIR}' ERR
+
+echo "Creating source package"
+
+# create a temporary git clone to ensure that we have a pristine source release
+git clone ${PROJECT_ROOT} ${CLONE_DIR}
+
+cd ${CLONE_DIR}
+JAVA_ROOT="paimon_python_java/paimon-python-java-bridge"
+rsync -a \
+  --exclude ".DS_Store" --exclude ".asf.yaml" --exclude ".git" \
+  --exclude ".github" --exclude ".gitignore" --exclude ".idea" \
+  --exclude ".mypy_cache" --exclude ".tox" --exclude "__pycache__" \
+  --exclude "build" --exclude "dist" --exclude "*.egg-info" \
+  --exclude "dev/.conda" --exclude "dev/.stage.txt" \
+  --exclude "dev/download" --exclude "dev/log" --exclude "**/__pycache__" \
+  --exclude "${JAVA_ROOT}/dependency-reduced-pom.xml" \
+  --exclude "${JAVA_ROOT}/target" \
+  . paimon-python-${RELEASE_VERSION}
+
+TAR czf ${RELEASE_DIR}/apache-paimon-python-${RELEASE_VERSION}-src.tgz paimon-python-${RELEASE_VERSION}
+gpg --armor --detach-sig ${RELEASE_DIR}/apache-paimon-python-${RELEASE_VERSION}-src.tgz
+cd ${RELEASE_DIR}
+${SHASUM} apache-paimon-python-${RELEASE_VERSION}-src.tgz > apache-paimon-python-${RELEASE_VERSION}-src.tgz.sha512
+
+rm -rf ${CLONE_DIR}
+
+echo "Done. Source release package and signatures created under ${RELEASE_DIR}/."
+
+cd ${CURR_DIR}

tools/releasing/update_branch_version.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Required variables
+##
+NEW_VERSION=${NEW_VERSION}
+
+if [ -z "${NEW_VERSION}" ]; then
+    echo "NEW_VERSION was not set."
+    exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+
+CURR_DIR=`pwd`
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+PROJECT_ROOT="${BASE_DIR}/../../"
+SETUP_UTILS_DIR="${PROJECT_ROOT}/setup_utils"
+
+# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
+if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
+    echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
+    exit 1
+fi
+
+###########################
+
+cd ${SETUP_UTILS_DIR}/
+
+# change version
+perl -pi -e "s#^__version__ = \".*\"#__version__ = \"${NEW_VERSION}\"#" version.py
+git commit -am "[release] Update version to ${NEW_VERSION}"
+
+NEW_VERSION_COMMIT_HASH=`git rev-parse HEAD`
+
+echo "Done. Created a new commit for the new version ${NEW_VERSION}, with hash ${NEW_VERSION_COMMIT_HASH}"
+echo "If this is a new version to be released (or a candidate to be voted on), don't forget to create a signed release tag on GitHub and push the changes."
+
+cd ${CURR_DIR}