Compiler search uses a pool of workers (spack#10190)

- spack.compilers.find_compilers now uses a multiprocess.pool.ThreadPool to execute
  system commands for the detection of compiler versions.

- A few memoized functions have been introduced to avoid poking the filesystem multiple
  times for the same results.

- Performance is much improved, and Spack no longer fork-bombs the system when doing a `compiler find`

lib/spack/llnl/util/filesystem.py

@@ -21,7 +21,7 @@
 
 import six
 from llnl.util import tty
-from llnl.util.lang import dedupe
+from llnl.util.lang import dedupe, memoized
 from spack.util.executable import Executable
 
 __all__ = [
@@ -1351,3 +1351,63 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     libraries = ['{0}.{1}'.format(lib, suffix) for lib in libraries]
 
     return LibraryList(find(root, libraries, recursive))
+
+
+@memoized
+def can_access_dir(path):
+    """Returns True if the argument is an accessible directory.
+
+    Args:
+        path: path to be tested
+
+    Returns:
+        True if ``path`` is an accessible directory, else False
+    """
+    return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK)
+
+
+@memoized
+def files_in(*search_paths):
+    """Returns all the files in paths passed as arguments.
+
+    Caller must ensure that each path in ``search_paths`` is a directory.
+
+    Args:
+        *search_paths: directories to be searched
+
+    Returns:
+        List of (file, full_path) tuples with all the files found.
+    """
+    files = []
+    for d in filter(can_access_dir, search_paths):
+        files.extend(filter(
+            lambda x: os.path.isfile(x[1]),
+            [(f, os.path.join(d, f)) for f in os.listdir(d)]
+        ))
+    return files
+
+
+def search_paths_for_executables(*path_hints):
+    """Given a list of path hints returns a list of paths where
+    to search for an executable.
+
+    Args:
+        *path_hints (list of paths): list of paths taken into
+            consideration for a search
+
+    Returns:
+        A list containing the real path of every existing directory
+        in `path_hints` and its `bin` subdirectory if it exists.
+    """
+    executable_paths = []
+    for path in path_hints:
+        if not os.path.isdir(path):
+            continue
+
+        executable_paths.append(path)
+
+        bin_dir = os.path.join(path, 'bin')
+        if os.path.isdir(bin_dir):
+            executable_paths.append(bin_dir)
+
+    return executable_paths

lib/spack/llnl/util/multiproc.py

@@ -8,25 +8,9 @@
 than multiprocessing.Pool.apply() can.  For example, apply() will fail
 to pickle functions if they're passed indirectly as parameters.
 """
-from multiprocessing import Process, Pipe, Semaphore, Value
+from multiprocessing import Semaphore, Value
 
-__all__ = ['spawn', 'parmap', 'Barrier']
-
-
-def spawn(f):
-    def fun(pipe, x):
-        pipe.send(f(x))
-        pipe.close()
-    return fun
-
-
-def parmap(f, elements):
-    pipe = [Pipe() for x in elements]
-    proc = [Process(target=spawn(f), args=(c, x))
-            for x, (p, c) in zip(elements, pipe)]
-    [p.start() for p in proc]
-    [p.join() for p in proc]
-    return [p.recv() for (p, c) in pipe]
+__all__ = ['Barrier']
 
 
 class Barrier:

lib/spack/llnl/util/tty/__init__.py

@@ -3,14 +3,17 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from datetime import datetime
+from __future__ import unicode_literals
+
 import fcntl
 import os
 import struct
 import sys
 import termios
 import textwrap
 import traceback
+import six
+from datetime import datetime
 from six import StringIO
 from six.moves import input
 
@@ -155,7 +158,7 @@ def msg(message, *args, **kwargs):
         cwrite("@*b{%s==>} %s%s" % (
             st_text, get_timestamp(), cescape(message)))
     for arg in args:
-        print(indent + str(arg))
+        print(indent + six.text_type(arg))
 
 
 def info(message, *args, **kwargs):
@@ -172,17 +175,17 @@ def info(message, *args, **kwargs):
     if _stacktrace:
         st_text = process_stacktrace(st_countback)
     cprint("@%s{%s==>} %s%s" % (
-        format, st_text, get_timestamp(), cescape(str(message))),
-        stream=stream)
+        format, st_text, get_timestamp(), cescape(six.text_type(message))
+    ), stream=stream)
     for arg in args:
         if wrap:
             lines = textwrap.wrap(
-                str(arg), initial_indent=indent, subsequent_indent=indent,
-                break_long_words=break_long_words)
+                six.text_type(arg), initial_indent=indent,
+                subsequent_indent=indent, break_long_words=break_long_words)
             for line in lines:
                 stream.write(line + '\n')
         else:
-            stream.write(indent + str(arg) + '\n')
+            stream.write(indent + six.text_type(arg) + '\n')
 
 
 def verbose(message, *args, **kwargs):
@@ -204,7 +207,7 @@ def error(message, *args, **kwargs):
 
     kwargs.setdefault('format', '*r')
     kwargs.setdefault('stream', sys.stderr)
-    info("Error: " + str(message), *args, **kwargs)
+    info("Error: " + six.text_type(message), *args, **kwargs)
 
 
 def warn(message, *args, **kwargs):
@@ -213,7 +216,7 @@ def warn(message, *args, **kwargs):
 
     kwargs.setdefault('format', '*Y')
     kwargs.setdefault('stream', sys.stderr)
-    info("Warning: " + str(message), *args, **kwargs)
+    info("Warning: " + six.text_type(message), *args, **kwargs)
 
 
 def die(message, *args, **kwargs):
@@ -237,7 +240,7 @@ def get_number(prompt, **kwargs):
     while number is None:
         msg(prompt, newline=False)
         ans = input()
-        if ans == str(abort):
+        if ans == six.text_type(abort):
             return None
 
         if ans:
@@ -303,7 +306,7 @@ def hline(label=None, **kwargs):
         cols -= 2
     cols = min(max_width, cols)
 
-    label = str(label)
+    label = six.text_type(label)
     prefix = char * 2 + " "
     suffix = " " + (cols - len(prefix) - clen(label)) * char
 

lib/spack/llnl/util/tty/colify.py

@@ -6,7 +6,7 @@
 """
 Routines for printing columnar output.  See ``colify()`` for more information.
 """
-from __future__ import division
+from __future__ import division, unicode_literals
 
 import os
 import sys

lib/spack/llnl/util/tty/color.py

@@ -59,10 +59,14 @@
 
 To output an @, use '@@'.  To output a } inside braces, use '}}'.
 """
+from __future__ import unicode_literals
 import re
 import sys
+
 from contextlib import contextmanager
 
+import six
+
 
 class ColorParseError(Exception):
     """Raised when a color format fails to parse."""
@@ -244,7 +248,7 @@ def cescape(string):
     Returns:
         (str): the string with color codes escaped
     """
-    string = str(string)
+    string = six.text_type(string)
     string = string.replace('@', '@@')
     string = string.replace('}', '}}')
     return string

lib/spack/llnl/util/tty/log.py

@@ -5,6 +5,8 @@
 
 """Utility classes for logging the output of blocks of code.
 """
+from __future__ import unicode_literals
+
 import multiprocessing
 import os
 import re

lib/spack/spack/architecture.py

@@ -56,18 +56,15 @@
 attributes front_os and back_os. The operating system as described earlier,
 will be responsible for compiler detection.
 """
-import os
 import inspect
-import platform as py_platform
 
-import llnl.util.multiproc as mp
 import llnl.util.tty as tty
 from llnl.util.lang import memoized, list_modules, key_ordering
 
+import spack.compiler
 import spack.paths
 import spack.error as serr
 from spack.util.naming import mod_to_class
-from spack.util.environment import get_path
 from spack.util.spack_yaml import syaml_dict
 
 
@@ -229,100 +226,13 @@ def __repr__(self):
         return self.__str__()
 
     def _cmp_key(self):
-        return (self.name, self.version)
-
-    def find_compilers(self, *paths):
-        """
-        Return a list of compilers found in the supplied paths.
-        This invokes the find() method for each Compiler class,
-        and appends the compilers detected to a list.
-        """
-        if not paths:
-            paths = get_path('PATH')
-        # Make sure path elements exist, and include /bin directories
-        # under prefixes.
-        filtered_path = []
-        for p in paths:
-            # Eliminate symlinks and just take the real directories.
-            p = os.path.realpath(p)
-            if not os.path.isdir(p):
-                continue
-            filtered_path.append(p)
-
-            # Check for a bin directory, add it if it exists
-            bin = os.path.join(p, 'bin')
-            if os.path.isdir(bin):
-                filtered_path.append(os.path.realpath(bin))
-
-        # Once the paths are cleaned up, do a search for each type of
-        # compiler.  We can spawn a bunch of parallel searches to reduce
-        # the overhead of spelunking all these directories.
-        # NOTE: we import spack.compilers here to avoid init order cycles
-        import spack.compilers
-        types = spack.compilers.all_compiler_types()
-        compiler_lists = mp.parmap(
-            lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path),
-            types)
-
-        # ensure all the version calls we made are cached in the parent
-        # process, as well.  This speeds up Spack a lot.
-        clist = [comp for cl in compiler_lists for comp in cl]
-        return clist
-
-    def find_compiler(self, cmp_cls, *path):
-        """Try to find the given type of compiler in the user's
-           environment. For each set of compilers found, this returns
-           compiler objects with the cc, cxx, f77, fc paths and the
-           version filled in.
-
-           This will search for compilers with the names in cc_names,
-           cxx_names, etc. and it will group them if they have common
-           prefixes, suffixes, and versions.  e.g., gcc-mp-4.7 would
-           be grouped with g++-mp-4.7 and gfortran-mp-4.7.
-        """
-        dicts = mp.parmap(
-            lambda t: cmp_cls._find_matches_in_path(*t),
-            [(cmp_cls.cc_names,  cmp_cls.cc_version)  + tuple(path),
-             (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path),
-             (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path),
-             (cmp_cls.fc_names,  cmp_cls.fc_version)  + tuple(path)])
-
-        all_keys = set()
-        for d in dicts:
-            all_keys.update(d)
-
-        compilers = {}
-        for k in all_keys:
-            ver, pre, suf = k
-
-            # Skip compilers with unknown version.
-            if ver == 'unknown':
-                continue
-
-            paths = tuple(pn[k] if k in pn else None for pn in dicts)
-            spec = spack.spec.CompilerSpec(cmp_cls.name, ver)
-
-            if ver in compilers:
-                prev = compilers[ver]
-
-                # prefer the one with more compilers.
-                prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
-                newcount = len([p for p in paths if p is not None])
-                prevcount = len([p for p in prev_paths if p is not None])
-
-                # Don't add if it's not an improvement over prev compiler.
-                if newcount <= prevcount:
-                    continue
-
-            compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths)
-
-        return list(compilers.values())
+        return self.name, self.version
 
     def to_dict(self):
-        d = {}
-        d['name'] = self.name
-        d['version'] = self.version
-        return d
+        return {
+            'name': self.name,
+            'version': self.version
+        }
 
 
 @key_ordering

lib/spack/spack/cmd/compiler.py

@@ -79,7 +79,7 @@ def compiler_find(args):
     # Just let compiler_find do the
     # entire process and return an empty config from all_compilers
     # Default for any other process is init_config=True
-    compilers = [c for c in spack.compilers.find_compilers(*paths)]
+    compilers = [c for c in spack.compilers.find_compilers(paths)]
     new_compilers = []
     for c in compilers:
         arch_spec = ArchSpec(None, c.operating_system, c.target)