Merge pull request #2714 from pypa/feature/distutils-sync

Update distutils

changelog.d/2714.change.rst

@@ -0,0 +1 @@
+Update to distutils at pypa/distutils@e2627b7.

setuptools/_distutils/command/build.py

@@ -102,7 +102,7 @@ def finalize_options(self):
         # particular module distribution -- if user didn't supply it, pick
         # one of 'build_purelib' or 'build_platlib'.
         if self.build_lib is None:
-            if self.distribution.ext_modules:
+            if self.distribution.has_ext_modules():
                 self.build_lib = self.build_platlib
             else:
                 self.build_lib = self.build_purelib

setuptools/_distutils/command/build_ext.py

@@ -690,13 +690,15 @@ def get_export_symbols(self, ext):
         provided, "PyInit_" + module_name.  Only relevant on Windows, where
         the .pyd file (DLL) must export the module "PyInit_" function.
         """
-        suffix = '_' + ext.name.split('.')[-1]
+        name = ext.name.split('.')[-1]
         try:
             # Unicode module name support as defined in PEP-489
             # https://www.python.org/dev/peps/pep-0489/#export-hook-name
-            suffix.encode('ascii')
+            name.encode('ascii')
         except UnicodeEncodeError:
-            suffix = 'U' + suffix.encode('punycode').replace(b'-', b'_').decode('ascii')
+            suffix = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii')
+        else:
+            suffix = "_" + name
 
         initfunc_name = "PyInit" + suffix
         if initfunc_name not in ext.export_symbols:

setuptools/_distutils/command/install.py

@@ -348,7 +348,7 @@ def finalize_options(self):
         # module distribution is pure or not.  Of course, if the user
         # already specified install_lib, use their selection.
         if self.install_lib is None:
-            if self.distribution.ext_modules: # has extensions: non-pure
+            if self.distribution.has_ext_modules(): # has extensions: non-pure
                 self.install_lib = self.install_platlib
             else:
                 self.install_lib = self.install_purelib

setuptools/_distutils/cygwinccompiler.py

@@ -44,6 +44,8 @@
 #   (ld supports -shared)
 # * mingw gcc 3.2/ld 2.13 works
 #   (ld supports -shared)
+# * llvm-mingw with Clang 11 works
+#   (lld supports -shared)
 
 import os
 import sys
@@ -109,41 +111,46 @@ def __init__(self, verbose=0, dry_run=0, force=0):
                 "Compiling may fail because of undefined preprocessor macros."
                 % details)
 
-        self.gcc_version, self.ld_version, self.dllwrap_version = \
-            get_versions()
-        self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" %
-                         (self.gcc_version,
-                          self.ld_version,
-                          self.dllwrap_version) )
-
-        # ld_version >= "2.10.90" and < "2.13" should also be able to use
-        # gcc -mdll instead of dllwrap
-        # Older dllwraps had own version numbers, newer ones use the
-        # same as the rest of binutils ( also ld )
-        # dllwrap 2.10.90 is buggy
-        if self.ld_version >= "2.10.90":
-            self.linker_dll = "gcc"
-        else:
-            self.linker_dll = "dllwrap"
+        self.cc = os.environ.get('CC', 'gcc')
+        self.cxx = os.environ.get('CXX', 'g++')
+
+        if ('gcc' in self.cc): # Start gcc workaround
+            self.gcc_version, self.ld_version, self.dllwrap_version = \
+                get_versions()
+            self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" %
+                             (self.gcc_version,
+                              self.ld_version,
+                              self.dllwrap_version) )
+
+            # ld_version >= "2.10.90" and < "2.13" should also be able to use
+            # gcc -mdll instead of dllwrap
+            # Older dllwraps had own version numbers, newer ones use the
+            # same as the rest of binutils ( also ld )
+            # dllwrap 2.10.90 is buggy
+            if self.ld_version >= "2.10.90":
+                self.linker_dll = self.cc
+            else:
+                self.linker_dll = "dllwrap"
 
-        # ld_version >= "2.13" support -shared so use it instead of
-        # -mdll -static
-        if self.ld_version >= "2.13":
+            # ld_version >= "2.13" support -shared so use it instead of
+            # -mdll -static
+            if self.ld_version >= "2.13":
+                shared_option = "-shared"
+            else:
+                shared_option = "-mdll -static"
+        else: # Assume linker is up to date
+            self.linker_dll = self.cc
             shared_option = "-shared"
-        else:
-            shared_option = "-mdll -static"
 
-        # Hard-code GCC because that's what this is all about.
-        # XXX optimization, warnings etc. should be customizable.
-        self.set_executables(compiler='gcc -mcygwin -O -Wall',
-                             compiler_so='gcc -mcygwin -mdll -O -Wall',
-                             compiler_cxx='g++ -mcygwin -O -Wall',
-                             linker_exe='gcc -mcygwin',
+        self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc,
+                             compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc,
+                             compiler_cxx='%s -mcygwin -O -Wall' % self.cxx,
+                             linker_exe='%s -mcygwin' % self.cc,
                              linker_so=('%s -mcygwin %s' %
                                         (self.linker_dll, shared_option)))
 
         # cygwin and mingw32 need different sets of libraries
-        if self.gcc_version == "2.91.57":
+        if ('gcc' in self.cc and self.gcc_version == "2.91.57"):
             # cygwin shouldn't need msvcrt, but without the dlls will crash
             # (gcc version 2.91.57) -- perhaps something about initialization
             self.dll_libraries=["msvcrt"]
@@ -281,26 +288,26 @@ def __init__(self, verbose=0, dry_run=0, force=0):
 
         # ld_version >= "2.13" support -shared so use it instead of
         # -mdll -static
-        if self.ld_version >= "2.13":
-            shared_option = "-shared"
-        else:
+        if ('gcc' in self.cc and self.ld_version < "2.13"):
             shared_option = "-mdll -static"
+        else:
+            shared_option = "-shared"
 
         # A real mingw32 doesn't need to specify a different entry point,
         # but cygwin 2.91.57 in no-cygwin-mode needs it.
-        if self.gcc_version <= "2.91.57":
+        if ('gcc' in self.cc and self.gcc_version <= "2.91.57"):
             entry_point = '--entry _DllMain@12'
         else:
             entry_point = ''
 
-        if is_cygwingcc():
+        if is_cygwincc(self.cc):
             raise CCompilerError(
                 'Cygwin gcc cannot be used with --compiler=mingw32')
 
-        self.set_executables(compiler='gcc -O -Wall',
-                             compiler_so='gcc -mdll -O -Wall',
-                             compiler_cxx='g++ -O -Wall',
-                             linker_exe='gcc',
+        self.set_executables(compiler='%s -O -Wall' % self.cc,
+                             compiler_so='%s -mdll -O -Wall' % self.cc,
+                             compiler_cxx='%s -O -Wall' % self.cxx,
+                             linker_exe='%s' % self.cc,
                              linker_so='%s %s %s'
                                         % (self.linker_dll, shared_option,
                                            entry_point))
@@ -351,6 +358,10 @@ def check_config_h():
     if "GCC" in sys.version:
         return CONFIG_H_OK, "sys.version mentions 'GCC'"
 
+    # Clang would also work
+    if "Clang" in sys.version:
+        return CONFIG_H_OK, "sys.version mentions 'Clang'"
+
     # let's see if __GNUC__ is mentioned in python.h
     fn = sysconfig.get_config_h_filename()
     try:
@@ -397,7 +408,7 @@ def get_versions():
     commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version']
     return tuple([_find_exe_version(cmd) for cmd in commands])
 
-def is_cygwingcc():
-    '''Try to determine if the gcc that would be used is from cygwin.'''
-    out_string = check_output(['gcc', '-dumpmachine'])
+def is_cygwincc(cc):
+    '''Try to determine if the compiler that would be used is from cygwin.'''
+    out_string = check_output([cc, '-dumpmachine'])
     return out_string.strip().endswith(b'cygwin')

setuptools/_distutils/filelist.py

@@ -4,13 +4,16 @@
 and building lists of files.
 """
 
-import os, re
+import os
+import re
 import fnmatch
 import functools
+
 from distutils.util import convert_path
 from distutils.errors import DistutilsTemplateError, DistutilsInternalError
 from distutils import log
 
+
 class FileList:
     """A list of files built by on exploring the filesystem and filtered by
     applying various patterns to what we find there.
@@ -46,7 +49,7 @@ def debug_print(self, msg):
         if DEBUG:
             print(msg)
 
-    # -- List-like methods ---------------------------------------------
+    # Collection methods
 
     def append(self, item):
         self.files.append(item)
@@ -61,17 +64,15 @@ def sort(self):
         for sort_tuple in sortable_files:
             self.files.append(os.path.join(*sort_tuple))
 
-
-    # -- Other miscellaneous utility methods ---------------------------
+    # Other miscellaneous utility methods
 
     def remove_duplicates(self):
         # Assumes list has been sorted!
         for i in range(len(self.files) - 1, 0, -1):
             if self.files[i] == self.files[i - 1]:
                 del self.files[i]
 
-
-    # -- "File template" methods ---------------------------------------
+    # "File template" methods
 
     def _parse_template_line(self, line):
         words = line.split()
@@ -146,9 +147,11 @@ def process_template_line(self, line):
                              (dir, ' '.join(patterns)))
             for pattern in patterns:
                 if not self.include_pattern(pattern, prefix=dir):
-                    log.warn(("warning: no files found matching '%s' "
-                                "under directory '%s'"),
-                             pattern, dir)
+                    msg = (
+                        "warning: no files found matching '%s' "
+                        "under directory '%s'"
+                    )
+                    log.warn(msg, pattern, dir)
 
         elif action == 'recursive-exclude':
             self.debug_print("recursive-exclude %s %s" %
@@ -174,8 +177,7 @@ def process_template_line(self, line):
             raise DistutilsInternalError(
                   "this cannot happen: invalid action '%s'" % action)
 
-
-    # -- Filtering/selection methods -----------------------------------
+    # Filtering/selection methods
 
     def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
         """Select strings (presumably filenames) from 'self.files' that
@@ -219,9 +221,8 @@ def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
                 files_found = True
         return files_found
 
-
-    def exclude_pattern (self, pattern,
-                         anchor=1, prefix=None, is_regex=0):
+    def exclude_pattern(
+            self, pattern, anchor=1, prefix=None, is_regex=0):
         """Remove strings (presumably filenames) from 'files' that match
         'pattern'.  Other parameters are the same as for
         'include_pattern()', above.
@@ -240,21 +241,47 @@ def exclude_pattern (self, pattern,
         return files_found
 
 
-# ----------------------------------------------------------------------
 # Utility functions
 
 def _find_all_simple(path):
     """
     Find all files under 'path'
     """
+    all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True))
     results = (
         os.path.join(base, file)
-        for base, dirs, files in os.walk(path, followlinks=True)
+        for base, dirs, files in all_unique
         for file in files
     )
     return filter(os.path.isfile, results)
 
 
+class _UniqueDirs(set):
+    """
+    Exclude previously-seen dirs from walk results,
+    avoiding infinite recursion.
+    Ref https://bugs.python.org/issue44497.
+    """
+    def __call__(self, walk_item):
+        """
+        Given an item from an os.walk result, determine
+        if the item represents a unique dir for this instance
+        and if not, prevent further traversal.
+        """
+        base, dirs, files = walk_item
+        stat = os.stat(base)
+        candidate = stat.st_dev, stat.st_ino
+        found = candidate in self
+        if found:
+            del dirs[:]
+        self.add(candidate)
+        return not found
+
+    @classmethod
+    def filter(cls, items):
+        return filter(cls(), items)
+
+
 def findall(dir=os.curdir):
     """
     Find all files under 'dir' and return the list of full filenames.
@@ -319,7 +346,8 @@ def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
         if os.sep == '\\':
             sep = r'\\'
         pattern_re = pattern_re[len(start): len(pattern_re) - len(end)]
-        pattern_re = r'%s\A%s%s.*%s%s' % (start, prefix_re, sep, pattern_re, end)
+        pattern_re = r'%s\A%s%s.*%s%s' % (
+            start, prefix_re, sep, pattern_re, end)
     else:                               # no prefix -- respect anchor flag
         if anchor:
             pattern_re = r'%s\A%s' % (start, pattern_re[len(start):])

setuptools/_distutils/spawn.py

@@ -15,11 +15,6 @@
 from distutils import log
 
 
-if sys.platform == 'darwin':
-    _cfg_target = None
-    _cfg_target_split = None
-
-
 def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None):
     """Run another program, specified as a command list 'cmd', in a new process.
 
@@ -52,28 +47,10 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None):
     env = env if env is not None else dict(os.environ)
 
     if sys.platform == 'darwin':
-        global _cfg_target, _cfg_target_split
-        if _cfg_target is None:
-            from distutils import sysconfig
-            _cfg_target = sysconfig.get_config_var(
-                                  'MACOSX_DEPLOYMENT_TARGET') or ''
-            if _cfg_target:
-                _cfg_target_split = [int(x) for x in _cfg_target.split('.')]
-        if _cfg_target:
-            # Ensure that the deployment target of the build process is not
-            # less than 10.3 if the interpreter was built for 10.3 or later.
-            # This ensures extension modules are built with correct
-            # compatibility values, specifically LDSHARED which can use
-            # '-undefined dynamic_lookup' which only works on >= 10.3.
-            cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
-            cur_target_split = [int(x) for x in cur_target.split('.')]
-            if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]:
-                my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
-                          'now "%s" but "%s" during configure;'
-                          'must use 10.3 or later'
-                                % (cur_target, _cfg_target))
-                raise DistutilsPlatformError(my_msg)
-            env.update(MACOSX_DEPLOYMENT_TARGET=cur_target)
+        from distutils.util import MACOSX_VERSION_VAR, get_macosx_target_ver
+        macosx_target_ver = get_macosx_target_ver()
+        if macosx_target_ver:
+            env[MACOSX_VERSION_VAR] = macosx_target_ver
 
     try:
         proc = subprocess.Popen(cmd, env=env)

setuptools/_distutils/tests/test_build_ext.py

@@ -316,7 +316,7 @@ def test_unicode_module_names(self):
         self.assertRegex(cmd.get_ext_filename(modules[0].name), r'foo(_d)?\..*')
         self.assertRegex(cmd.get_ext_filename(modules[1].name), r'föö(_d)?\..*')
         self.assertEqual(cmd.get_export_symbols(modules[0]), ['PyInit_foo'])
-        self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_gkaa'])
+        self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_1gaa'])
 
     def test_compiler_option(self):
         # cmd.compiler is an option and

setuptools/_distutils/tests/test_filelist.py

@@ -331,6 +331,16 @@ def test_non_local_discovery(self):
             expected = [file1]
             self.assertEqual(filelist.findall(temp_dir), expected)
 
+    @os_helper.skip_unless_symlink
+    def test_symlink_loop(self):
+        with os_helper.temp_dir() as temp_dir:
+            link = os.path.join(temp_dir, 'link-to-parent')
+            content = os.path.join(temp_dir, 'somefile')
+            os_helper.create_empty_file(content)
+            os.symlink('.', link)
+            files = filelist.findall(temp_dir)
+            assert len(files) == 1
+
 
 def test_suite():
     return unittest.TestSuite([