Merge pull request #1929 from Kami/migrate_from_setuppy_to_pyproject_…

…toml

Migrate from setup.py to pyproject.toml

.github/workflows/main.yml

@@ -195,7 +195,11 @@ jobs:
         run: |
           pip install -r requirements-ci.txt
 
-      - name: Run Checks
+      - name: Run shellcheck
+        run: |
+          shellcheck dist/*.sh contrib/*.sh
+
+      - name: Run Python Checks
         run: |
           tox -e black-check,isort-check,pyupgrade,checks,import-timings,lint,pylint
 

CHANGES.rst

@@ -108,6 +108,22 @@ Other
   2016.
   [Tomaz Muraus - @Kami]
 
+- Packaging related metadata has been migrated from ``setup.py`` to
+  ``pyproject.yaml`` file.
+  (GITHUB-1929)
+  [Tomaz Muraus - @Kami]
+
+- Deprecated and unsafe ``setup.py test`` convenience alias for running tests
+  using pytest has been removed in favor of running pytest directly.
+  (GITHUB-1929)
+  [Tomaz Muraus - @Kami]
+
+- Script for building release artifacts has been updated to utilize ``build``
+  Python package to build release artifacts (sdist + wheel) in an isolated
+  environment.
+  (GITHUB-1929)
+  [Tomaz Muraus - @Kami]
+
 Changes in Apache Libcloud 3.7.0
 --------------------------------
 
@@ -126,7 +142,7 @@ Compute
 
 - [CloudSigma] Update API URLs for US locations.
   (GITHUB-1781)
-  [Mohsen Hassani - @ mohsen-hassani-cs]
+  [Mohsen Hassani - @mohsen-hassani-cs]
 
 - [GCP] Fix OAuth2 desktop client login.
   (GITHUB-1806, GITHUB-1807)

MANIFEST.in

@@ -1,3 +1,5 @@
+global-exclude *.py[cod]
+global-exclude .pytest_cache
 include LICENSE
 include NOTICE
 include example_*.py
@@ -8,21 +10,10 @@ include pyproject.toml
 include requirements-tests.txt
 include requirements-lint.txt
 include libcloud/data/pricing.json
-prune libcloud/test/secrets.py
-prune requirements-rtd.txt
 include demos/*
 include scripts/check_file_names.sh
-include libcloud/test/*.py
-include libcloud/test/pricing_test.json
-include libcloud/test/secrets.py-dist
-include libcloud/test/common/*.py
-include libcloud/test/compute/*.py
-include libcloud/test/storage/*.py
-include libcloud/test/loadbalancer/*.py
-include libcloud/test/dns/*.py
-include libcloud/test/common/fixtures/*/*
-include libcloud/test/compute/fixtures/*/*
-include libcloud/test/compute/fixtures/*/*/*
-include libcloud/test/storage/fixtures/*/*
-include libcloud/test/loadbalancer/fixtures/*/*
-include libcloud/test/dns/fixtures/*/*
+recursive-exclude libcloud/test secrets.py
+prune libcloud/test/secrets.py
+prune requirements-rtd.txt
+prune dist
+prune build

NOTICE

@@ -1,5 +1,5 @@
 Apache Libcloud
-Copyright (c) 2010-2020 The Apache Software Foundation
+Copyright (c) 2010-2023 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

contrib/pre-commit.sh

@@ -15,15 +15,18 @@
 #  KIND, either express or implied.  See the License for the
 #  specific language governing permissions and limitations
 #  under the License.
+set -e
 
 files=$(git diff --cached --name-status | grep -v ^D | awk '$1 $2 { print $2}' | grep -e .py$)
+# shellcheck disable=SC2206
 array=(${files/// })
 
 for file in "${array[@]}"
 do
+    echo "Processing: ${file}"
     if [[ ${file} =~ "libcloud/test/" ]]; then
-        flake8 --max-line-length=160 ${file}
+        flake8 --max-line-length=160 "${file}"
     else
-        flake8 ${file}
+        flake8 "${file}"
     fi
 done

dist/deploy.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,15 +13,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+set -e
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 
-cd ..
+pushd "${SCRIPT_DIR}/../"
 
-VERSION=`python setup.py --version`
+# We redirect stderr to /dev/null since sometimes setuptools may print pyproject
+# related warning
+VERSION=$(python setup.py --version 2> /dev/null)
+popd
 
-cd dist
+pushd "${SCRIPT_DIR}"
 
 echo "Uploading packages"
-ls *$VERSION*.tar.gz *$VERSION*.whl *$VERSION*.tar.gz.asc
+# shellcheck disable=SC2086
+ls ./*$VERSION*.tar.gz ./*$VERSION*.whl ./*$VERSION*.tar.gz.asc
+# shellcheck disable=SC2086
+twine upload ./*$VERSION*.tar.gz ./*$VERSION*.whl ./*$VERSION*.tar.gz.asc
 
-twine upload *$VERSION*.tar.gz *$VERSION*.whl *$VERSION*.tar.gz.asc
+popd

dist/release.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,9 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 set -e
 
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+
 user=""
 case "$1" in
   -u)
@@ -30,12 +31,15 @@ if test -z "${user}"; then
   exit 1
 fi
 
-cd ..
+pushd "${SCRIPT_DIR}/../"
+
+python -m build
+
+popd
 
-python setup.py sdist --formats=bztar,zip,gztar
-python setup.py bdist_wheel
+pushd "${SCRIPT_DIR}"
 
-cd dist
+./hash.py ./*.tar.gz ./*.whl
+./sign.sh -u "${user}" ./*.tar.gz ./*.whl
 
-./hash.py *.tar.bz2 *.tar.gz *.zip *.whl
-./sign.sh -u ${user} *.tar.bz2 *.tar.gz *.zip *.whl
+popd

dist/sign.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -18,6 +18,7 @@
 #
 # USAGE: sign.sh -u user file1 file2 ...
 #
+set -e
 
 user=""
 case "$1" in
@@ -32,8 +33,8 @@ allfiles=$*
 
 
 
-gpg="`which gpg 2> /dev/null | head -1`"
-pgp="`which pgp 2> /dev/null | head -1`"
+gpg=$(which gpg 2> /dev/null | head -1)
+pgp=$(which pgp 2> /dev/null | head -1)
 
 echo "---------------------------------------------------------------------"
 echo ""
@@ -48,7 +49,8 @@ if test -x "${pgp}"; then
   for file in ${allfiles}; do
     if test -f "${file}"; then
       echo "pgp: creating asc signature file for ${file} ..."
-      ${pgp} -sba ${file} ${args}
+      # shellcheck disable=SC2086
+      "${pgp}" -sba "${file}" ${args}
     fi
   done
 # no pgp found - check for gpg
@@ -61,7 +63,8 @@ elif test -x "${gpg}"; then
   for file in ${allfiles}; do
     if test -f "${file}"; then
       echo "gpg: creating asc signature file for ${file} ..."
-      ${gpg} --armor ${args} --detach-sign ${file}
+      # shellcheck disable=SC2086
+      "${gpg}" --armor ${args} --detach-sign "${file}"
     fi
   done
 else

dist/verify_checksums.sh

@@ -21,7 +21,7 @@ VERSION=$1
 
 if [ ! "${VERSION}" ]; then
     echo "Usage: ${0} <version name>"
-    echo "For example: ${0} apache-libcloud-3.4.0"
+    echo "For example: ${0} apache-libcloud-3.7.0"
     exit 1
 fi
 
@@ -70,7 +70,8 @@ do
     file_name="${VERSION}${extension}"
 
     if [ "${extension}" = "-py2.py3-none-any.whl" ]; then
-        file_name=$(echo ${file_name} | sed "s/apache-libcloud/apache_libcloud/g")
+        # shellcheck disable=SC2001
+        file_name=$(echo "${file_name}" | sed "s/apache-libcloud/apache_libcloud/g")
     fi
 
     apache_url="${APACHE_MIRROR_URL}/${file_name}"
@@ -97,6 +98,7 @@ do
     echo "Downloading file from Apache mirror..."
     wget --quiet "${apache_url}" -O "${file_path_apache}"
 
+    # shellcheck disable=SC2181
     if [ $? -ne 0 ]; then
         echo "[ERR] Failed to download file: ${apache_url}"
         exit 2
@@ -105,6 +107,7 @@ do
     echo "Downloading file from PyPi mirror..."
     wget --quiet "${pypi_url}" -O "${file_path_pypi}"
 
+    # shellcheck disable=SC2181
     if [ $? -ne 0 ]; then
         echo "[ERR] Failed to download file: ${pypi_url}"
         exit 2
@@ -113,8 +116,8 @@ do
     sha512sum_apache=$(sha512sum "${file_path_apache}" | awk '{ print $1 }')
     sha512sum_pypi=$(sha512sum "${file_path_pypi}"| awk '{ print $1 }')
 
-    if [ ${sha512sum_apache} != ${sha512sum_pypi} ]; then
-       echo "[ERROR] SHA512 sum for file ${file_name} doesn\'t match"
+    if [ "${sha512sum_apache}" != "${sha512sum_pypi}" ]; then
+       echo "[ERROR] SHA512 sum for file ${file_name} doesn't match"
        echo ""
        echo "${file_name_apache}: ${sha512sum_apache}"
        echo "${file_name_pypi}: ${sha512sum_pypi}"

docs/committer_guide.rst

@@ -97,24 +97,27 @@ preparing a release.
 
 .. note::
 
-  It's important that you have the latest versions of ``setuptools``, ``wheel``
-  and ``pip`` installed to ensure the generated wheel files contain correct
-  metadata.
+  It's important that you have the latest versions of ``build`` package
+  installed to ensure the generated wheel files contain correct metadata.
 
-We have a script that runs the required setup.py commands and then hashes
-and signs the files. You will need the latest version of ``pip`` and the ``wheel``
-package. To run it:
+We have a script that runs the required commands and then hashes and signs the
+files. You will need the latest version of ``build`` package.
+
+To run it:
 
 .. sourcecode:: bash
 
+    # Install build dependencies
+    pip install -e ".[build]"
+
     cd dist
     ./release.sh -u <yourusername>@apache.org
 
 ``-u`` argument will be used to find a key with a matching email address in
 your local GPG database.
 
 This should result in a set of
-``apache-libcloud-${VERSION}.{tar.bz2,tar.gz,zip,whl}{,asc,md5,sha1}`` files that
+``apache-libcloud-${VERSION}.{tar.gz,whl}{,asc,md5,sha1}`` files that
 are suitable to be uploaded for a release.
 
 Copy the artifacts in another directory, unpack one of them and test it with ``tox``.
@@ -173,11 +176,15 @@ key.
 7. Publishing package to PyPi
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-We have a script that runs uploads the signed Python source files to PyPi. It uses twine, so ensure
-you have twine available in your path `which twine` before running. Twine can be downloaded from https://pypi.python.org/pypi/twine
+We have a script that runs uploads the signed Python source files to PyPi. It
+uses twine, so ensure you have twine available in your path `which twine`
+before running. Twine can be downloaded from https://pypi.python.org/pypi/twine
 
 .. sourcecode:: bash
 
+    # Install publish dependencies
+    pip install -e ".[publish]"
+
     cd dist
     ./deploy.sh