bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)

Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform


https://bugs.python.org/issue29877

Author: virtuald

Lib/compileall.py

@@ -16,10 +16,6 @@
 import py_compile
 import struct
 
-try:
-    from concurrent.futures import ProcessPoolExecutor
-except ImportError:
-    ProcessPoolExecutor = None
 from functools import partial
 
 __all__ = ["compile_dir","compile_file","compile_path"]
@@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     workers:   maximum number of parallel workers
     invalidation_mode: how the up-to-dateness of the pyc will be checked
     """
-    if workers is not None and workers < 0:
-        raise ValueError('workers must be greater or equal to 0')
-
+    ProcessPoolExecutor = None
+    if workers is not None:
+        if workers < 0:
+            raise ValueError('workers must be greater or equal to 0')
+        elif workers != 1:
+            try:
+                # Only import when needed, as low resource platforms may
+                # fail to import it
+                from concurrent.futures import ProcessPoolExecutor
+            except ImportError:
+                workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True

Lib/test/test_compileall.py

@@ -167,7 +167,7 @@ def test_compile_dir_pathlike(self):
         self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
         self.assertTrue(os.path.isfile(self.bc_path))
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_pool_called(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(pool_mock.called)
@@ -177,19 +177,19 @@ def test_compile_workers_non_positive(self):
                                     "workers must be greater or equal to 0"):
             compileall.compile_dir(self.directory, workers=-1)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_workers_cpu_count(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=0)
         self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     @mock.patch('compileall.compile_file')
     def test_compile_one_worker(self, compile_file_mock, pool_mock):
         compileall.compile_dir(self.directory, quiet=True)
         self.assertFalse(pool_mock.called)
         self.assertTrue(compile_file_mock.called)
 
-    @mock.patch('compileall.ProcessPoolExecutor', new=None)
+    @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
     @mock.patch('compileall.compile_file')
     def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)

Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst

@@ -0,0 +1,2 @@
+compileall: import ProcessPoolExecutor only when needed, preventing hangs on
+low resource platforms