ARROW-12068: [Python] Stop using distutils

According to PEP 632, distutils will be deprecated in Python 3.10 and removed in 3.12.

* switch to `setuptools` for general packaging
* use the `Version` class from the `packaging` project instead of `distutils.LooseVersion`

Closes apache#9849 from pitrou/ARROW-12068-remove-distutils

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>

Author: pitrou

LICENSE.txt

@@ -2209,3 +2209,12 @@ The files in cpp/src/arrow/vendored/fast_float/ contain code from
 https://github.com/lemire/fast_float
 
 which is made available under the Apache License 2.0.
+
+--------------------------------------------------------------------------------
+
+The file python/pyarrow/vendored/version.py contains code from
+
+https://github.com/pypa/packaging/
+
+which is made available under both the Apache license v2.0 and the
+BSD 2-clause license.

dev/release/rat_exclude_files.txt

@@ -136,6 +136,7 @@ python/MANIFEST.in
 python/manylinux1/.dockerignore
 python/pyarrow/includes/__init__.pxd
 python/pyarrow/tests/__init__.py
+python/pyarrow/vendored/*
 python/requirements*.txt
 pax_global_header
 MANIFEST.in

dev/tasks/conda-recipes/azure.clean.yml

@@ -13,7 +13,7 @@ jobs:
     displayName: Clone arrow
 
   - script: |
-      conda install -y -c conda-forge pandas anaconda-client
+      conda install -y -c conda-forge pandas anaconda-client packaging
     displayName: Install requirements
 
   - script: |

dev/tasks/conda-recipes/clean.py

@@ -1,4 +1,3 @@
-from distutils.version import LooseVersion
 from subprocess import check_output, check_call
 from typing import List
 
@@ -7,6 +6,8 @@
 import pandas as pd
 import sys
 
+from packaging.version import Version
+
 
 VERSIONS_TO_KEEP = 5
 PACKAGES = [
@@ -44,7 +45,7 @@ def packages_to_delete(package_name: str, platform: str) -> List[str]:
         env=env,
     )
     pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name])
-    pkgs["version"] = pkgs["version"].map(LooseVersion)
+    pkgs["version"] = pkgs["version"].map(Version)
     pkgs["py_version"] = pkgs["build"].str.slice(0, 4)
 
     to_delete = []

python/examples/plasma/sorting/setup.py

@@ -16,7 +16,7 @@
 # under the License.
 
 import numpy as np
-from distutils.core import setup
+from setuptools import setup
 from Cython.Build import cythonize
 
 setup(

python/pyarrow/feather.py

@@ -22,10 +22,11 @@
 from pyarrow.lib import (Codec, FeatherError, Table,  # noqa
                          concat_tables, schema)
 import pyarrow.lib as ext
+from pyarrow.vendored.version import Version
 
 
 def _check_pandas_version():
-    if _pandas_api.loose_version < '0.17.0':
+    if _pandas_api.loose_version < Version('0.17.0'):
         raise ImportError("feather requires pandas >= 0.17.0")
 
 

python/pyarrow/pandas-shim.pxi

@@ -53,12 +53,13 @@ cdef class _PandasAPIShim(object):
             else:
                 return
 
+        from pyarrow.vendored.version import Version
+
         self._pd = pd
         self._version = pd.__version__
-        from distutils.version import LooseVersion
-        self._loose_version = LooseVersion(pd.__version__)
+        self._loose_version = Version(pd.__version__)
 
-        if self._loose_version < LooseVersion('0.23.0'):
+        if self._loose_version < Version('0.23.0'):
             self._have_pandas = False
             if raise_:
                 raise ImportError(
@@ -82,7 +83,7 @@ cdef class _PandasAPIShim(object):
             self._series, self._index, self._categorical_type,
             self._extension_array)
         self._extension_dtype = pd.api.extensions.ExtensionDtype
-        if self._loose_version >= LooseVersion('0.24.0'):
+        if self._loose_version >= Version('0.24.0'):
             self._is_extension_array_dtype = \
                 pd.api.types.is_extension_array_dtype
         else:
@@ -92,12 +93,12 @@ cdef class _PandasAPIShim(object):
         self._datetimetz_type = pd.api.types.DatetimeTZDtype
         self._have_pandas = True
 
-        if self._loose_version > LooseVersion('0.25'):
+        if self._loose_version > Version('0.25'):
             self.has_sparse = False
         else:
             self.has_sparse = True
 
-        self._pd024 = self._loose_version >= LooseVersion('0.24')
+        self._pd024 = self._loose_version >= Version('0.24')
 
     cdef inline _check_import(self, bint raise_=True):
         if self._tried_importing_pandas:

python/pyarrow/tests/parquet/test_dataset.py

@@ -17,7 +17,6 @@
 
 import datetime
 import os
-from distutils.version import LooseVersion
 
 import numpy as np
 import pytest
@@ -30,6 +29,7 @@
     parametrize_legacy_dataset, parametrize_legacy_dataset_fixed,
     parametrize_legacy_dataset_not_supported)
 from pyarrow.util import guid
+from pyarrow.vendored.version import Version
 
 try:
     import pyarrow.parquet as pq
@@ -633,7 +633,7 @@ def test_read_partitioned_directory_s3fs_wrapper(
 
     from pyarrow.filesystem import S3FSWrapper
 
-    if s3fs.__version__ >= LooseVersion("0.5"):
+    if Version(s3fs.__version__) >= Version("0.5"):
         pytest.skip("S3FSWrapper no longer working for s3fs 0.5+")
 
     fs, path = s3_example_s3fs

python/pyarrow/tests/parquet/test_pandas.py

@@ -17,7 +17,6 @@
 
 import io
 import json
-from distutils.version import LooseVersion
 
 import numpy as np
 import pytest
@@ -26,6 +25,7 @@
 from pyarrow.tests.parquet.common import (
     parametrize_legacy_dataset, parametrize_legacy_dataset_not_supported)
 from pyarrow.util import guid
+from pyarrow.vendored.version import Version
 
 try:
     import pyarrow.parquet as pq
@@ -559,7 +559,7 @@ def test_write_to_dataset_pandas_preserve_extensiondtypes(
     tempdir, use_legacy_dataset
 ):
     # ARROW-8251 - preserve pandas extension dtypes in roundtrip
-    if LooseVersion(pd.__version__) < "1.0.0":
+    if Version(pd.__version__) < Version("1.0.0"):
         pytest.skip("__arrow_array__ added to pandas in 1.0.0")
 
     df = pd.DataFrame({'part': 'a', "col": [1, 2, 3]})

python/pyarrow/tests/test_fs.py

@@ -16,7 +16,6 @@
 # under the License.
 
 from datetime import datetime, timezone, timedelta
-from distutils.version import LooseVersion
 import gzip
 import os
 import pathlib
@@ -28,6 +27,8 @@
 
 import pyarrow as pa
 from pyarrow.tests.test_io import assert_file_not_found
+from pyarrow.vendored.version import Version
+
 from pyarrow.fs import (FileType, FileInfo, FileSelector, FileSystem,
                         LocalFileSystem, SubTreeFileSystem, _MockFileSystem,
                         FileSystemHandler, PyFileSystem, FSSpecHandler)
@@ -355,7 +356,8 @@ def py_fsspec_memoryfs(request, tempdir):
 @pytest.fixture
 def py_fsspec_s3fs(request, s3_connection, s3_server):
     s3fs = pytest.importorskip("s3fs")
-    if sys.version_info < (3, 7) and s3fs.__version__ >= LooseVersion("0.5"):
+    if (sys.version_info < (3, 7) and
+            Version(s3fs.__version__) >= Version("0.5")):
         pytest.skip("s3fs>=0.5 version is async and requires Python >= 3.7")
 
     host, port, access_key, secret_key = s3_connection