Convert Python 2 idioms to Python 2/3-compatible ones.

- convert print, StringIO, except as, octals, izip
- convert print statement to print function
- convert StringIO to six.StringIO
  - remove usage of csv reader in Spec, in favor of simple regex
  - csv reader only does byte strings
- convert 0755 octal literals to 0o755
- convert `except Foo, e` to `except Foo as e`
- fix a few places `str` is used.
  - may need to switch everything to str later.
- convert iteritems usages to use six.iteritems
- fix urllib and HTMLParser
- port metaclasses to use six.with_metaclass
- More octal literal conversions for Python 2/3
- Fix a new octal literal.
- Convert `basestring` to `six.string_types`
- Convert xrange -> range
- Fix various issues with encoding, iteritems, and Python3 semantics.
- Convert contextlib.nested to explicitly nexted context managers.
- Convert use of filter() to list comprehensions.
- Replace reduce() with list comprehensions.
-  Clean up composite: replace inspect.ismethod() with callable()
- Python 3 doesn't have "method" objects; inspect.ismethod returns False.
- Need to use callable in Composite to make it work.
- Update colify to use future division.
- Fix zip() usages that need to be lists.
- Python3: Use line-buffered logging instead of unbuffered.
- Python3 raises an error with unbuffered I/O
  - See https://bugs.python.org/issue17404

bin/spack

@@ -24,6 +24,8 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
+
 import sys
 if (sys.version_info[0] > 2) or (sys.version_info[:2] < (2, 6)):
     v_info = sys.version_info[:3]
@@ -74,8 +76,8 @@ for pyc_file in orphaned_pyc_files:
     try:
         os.remove(pyc_file)
     except OSError as e:
-        print ("WARNING: Spack may fail mysteriously. "
-               "Couldn't remove orphaned .pyc file: %s" % pyc_file)
+        print("WARNING: Spack may fail mysteriously. "
+              "Couldn't remove orphaned .pyc file: %s" % pyc_file)
 
 # If there is no working directory, use the spack prefix.
 try:

lib/spack/llnl/util/filesystem.py

@@ -175,9 +175,9 @@ def change_sed_delimiter(old_delim, new_delim, *filenames):
 def set_install_permissions(path):
     """Set appropriate permissions on the installed file."""
     if os.path.isdir(path):
-        os.chmod(path, 0755)
+        os.chmod(path, 0o755)
     else:
-        os.chmod(path, 0644)
+        os.chmod(path, 0o644)
 
 
 def copy_mode(src, dest):

lib/spack/llnl/util/lang.py

@@ -27,6 +27,7 @@
 import functools
 import collections
 import inspect
+from six import string_types
 
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']
@@ -80,7 +81,7 @@ def index_by(objects, *funcs):
         return objects
 
     f = funcs[0]
-    if isinstance(f, basestring):
+    if isinstance(f, str):
         f = lambda x: getattr(x, funcs[0])
     elif isinstance(f, tuple):
         f = lambda x: tuple(getattr(x, p) for p in funcs[0])
@@ -326,7 +327,7 @@ def match_predicate(*args):
     """
     def match(string):
         for arg in args:
-            if isinstance(arg, basestring):
+            if isinstance(arg, string_types):
                 if re.search(arg, string):
                     return True
             elif isinstance(arg, list) or isinstance(arg, tuple):

lib/spack/llnl/util/tty/__init__.py

@@ -29,7 +29,7 @@
 import termios
 import struct
 import traceback
-from StringIO import StringIO
+from six import StringIO
 
 from llnl.util.tty.color import *
 
@@ -93,7 +93,7 @@ def msg(message, *args, **kwargs):
     else:
         cwrite("@*b{%s==>} %s" % (st_text, cescape(message)))
     for arg in args:
-        print indent + str(arg)
+        print(indent + str(arg))
 
 
 def info(message, *args, **kwargs):
@@ -201,7 +201,7 @@ def get_yes_or_no(prompt, **kwargs):
         if not ans:
             result = default_value
             if result is None:
-                print "Please enter yes or no."
+                print("Please enter yes or no.")
         else:
             if ans == 'y' or ans == 'yes':
                 result = True
@@ -239,7 +239,7 @@ def hline(label=None, **kwargs):
     out.write(label)
     out.write(suffix)
 
-    print out.getvalue()
+    print(out.getvalue())
 
 
 def terminal_size():

lib/spack/llnl/util/tty/colify.py

@@ -25,9 +25,11 @@
 """
 Routines for printing columnar output.  See colify() for more information.
 """
+from __future__ import division
+
 import os
 import sys
-from StringIO import StringIO
+from six import StringIO
 
 from llnl.util.tty import terminal_size
 from llnl.util.tty.color import clen, cextra
@@ -64,18 +66,18 @@ def config_variable_cols(elts, console_width, padding, cols=0):
     # Get a bound on the most columns we could possibly have.
     # 'clen' ignores length of ansi color sequences.
     lengths = [clen(e) for e in elts]
-    max_cols = max(1, console_width / (min(lengths) + padding))
+    max_cols = max(1, console_width // (min(lengths) + padding))
     max_cols = min(len(elts), max_cols)
 
     # Range of column counts to try.  If forced, use the supplied value.
-    col_range = [cols] if cols else xrange(1, max_cols + 1)
+    col_range = [cols] if cols else range(1, max_cols + 1)
 
     # Determine the most columns possible for the console width.
     configs = [ColumnConfig(c) for c in col_range]
     for i, length in enumerate(lengths):
         for conf in configs:
             if conf.valid:
-                col = i / ((len(elts) + conf.cols - 1) / conf.cols)
+                col = i // ((len(elts) + conf.cols - 1) // conf.cols)
                 p = padding if col < (conf.cols - 1) else 0
 
                 if conf.widths[col] < (length + p):
@@ -107,7 +109,7 @@ def config_uniform_cols(elts, console_width, padding, cols=0):
     # 'clen' ignores length of ansi color sequences.
     max_len = max(clen(e) for e in elts) + padding
     if cols == 0:
-        cols = max(1, console_width / max_len)
+        cols = max(1, console_width // max_len)
         cols = min(len(elts), cols)
 
     config = ColumnConfig(cols)
@@ -193,12 +195,12 @@ def colify(elts, **options):
         raise ValueError("method must be one of: " + allowed_methods)
 
     cols = config.cols
-    rows = (len(elts) + cols - 1) / cols
+    rows = (len(elts) + cols - 1) // cols
     rows_last_col = len(elts) % rows
 
-    for row in xrange(rows):
+    for row in range(rows):
         output.write(" " * indent)
-        for col in xrange(cols):
+        for col in range(cols):
             elt = col * rows + row
             width = config.widths[col] + cextra(elts[elt])
             if col < cols - 1:
@@ -233,7 +235,7 @@ def colify_table(table, **options):
     columns = len(table[0])
 
     def transpose():
-        for i in xrange(columns):
+        for i in range(columns):
             for row in table:
                 yield row[i]
 

lib/spack/llnl/util/tty/log.py

@@ -165,8 +165,12 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.p.join(60.0)  # 1 minute to join the child
 
     def _spawn_writing_daemon(self, read, input_stream):
-        # Parent: read from child, skip the with block.
-        read_file = os.fdopen(read, 'r', 0)
+        # This is the Parent: read from child, skip the with block.
+
+        # Use line buffering (3rd param = 1) since Python 3 has a bug
+        # that prevents unbuffered text I/O.
+        read_file = os.fdopen(read, 'r', 1)
+
         with open(self.filename, 'w') as log_file:
             with keyboard_input(input_stream):
                 while True:

lib/spack/spack/__init__.py

@@ -96,7 +96,7 @@
 try:
     repo = spack.repository.RepoPath()
     sys.meta_path.append(repo)
-except spack.error.SpackError, e:
+except spack.error.SpackError as e:
     tty.die('while initializing Spack RepoPath:', e.message)
 
 

lib/spack/spack/architecture.py

@@ -287,7 +287,7 @@ def find_compilers(self, *paths):
 
         # ensure all the version calls we made are cached in the parent
         # process, as well.  This speeds up Spack a lot.
-        clist = reduce(lambda x, y: x + y, compiler_lists)
+        clist = [comp for cl in compiler_lists for comp in cl]
         return clist
 
     def find_compiler(self, cmp_cls, *path):

lib/spack/spack/build_environment.py

@@ -57,6 +57,7 @@
 import shutil
 import sys
 import traceback
+from six import iteritems
 
 import llnl.util.lang as lang
 import llnl.util.tty as tty
@@ -310,7 +311,7 @@ def set_build_environment_variables(pkg, env, dirty=False):
     environment = compiler.environment
     if 'set' in environment:
         env_to_set = environment['set']
-        for key, value in env_to_set.iteritems():
+        for key, value in iteritems(env_to_set):
             env.set('SPACK_ENV_SET_%s' % key, value)
             env.set('%s' % key, value)
         # Let shell know which variables to set
@@ -322,8 +323,9 @@ def set_build_environment_variables(pkg, env, dirty=False):
         env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths)
 
     # Add bin directories from dependencies to the PATH for the build.
-    bin_dirs = reversed(filter(os.path.isdir, [
-        '%s/bin' % d.prefix for d in pkg.spec.dependencies(deptype='build')]))
+    bin_dirs = reversed(
+        [d.prefix.bin for d in pkg.spec.dependencies(deptype='build')
+         if os.path.isdir(d.prefix.bin)])
     bin_dirs = filter_system_bin_paths(bin_dirs)
     for item in bin_dirs:
         env.prepend_path('PATH', item)

lib/spack/spack/build_systems/autotools.py

@@ -146,7 +146,7 @@ def _do_patch_config_guess(self):
         if config_guess is not None:
             try:
                 check_call([config_guess], stdout=PIPE, stderr=PIPE)
-                mod = stat(my_config_guess).st_mode & 0777 | S_IWUSR
+                mod = stat(my_config_guess).st_mode & 0o777 | S_IWUSR
                 os.chmod(my_config_guess, mod)
                 shutil.copyfile(config_guess, my_config_guess)
                 return True

lib/spack/spack/cmd/__init__.py

@@ -22,6 +22,8 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
+
 import os
 import re
 import sys
@@ -186,7 +188,7 @@ def display_specs(specs, **kwargs):
     # Traverse the index and print out each package
     for i, (architecture, compiler) in enumerate(sorted(index)):
         if i > 0:
-            print
+            print()
 
         header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
                                       architecture, spack.spec.compiler_color,
@@ -205,7 +207,7 @@ def display_specs(specs, **kwargs):
 
             for abbrv, spec in zip(abbreviated, specs):
                 prefix = gray_hash(spec, hlen) if hashes else ''
-                print prefix + (format % (abbrv, spec.prefix))
+                print(prefix + (format % (abbrv, spec.prefix)))
 
         elif mode == 'deps':
             for spec in specs:

lib/spack/spack/cmd/arch.py

@@ -22,6 +22,8 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
+
 import spack.architecture as architecture
 
 description = "print architecture information about this machine"
@@ -36,6 +38,6 @@ def setup_parser(subparser):
 
 def arch(parser, args):
     if args.platform:
-        print architecture.platform()
+        print(architecture.platform())
     else:
-        print architecture.sys_type()
+        print(architecture.sys_type())

lib/spack/spack/cmd/common/arguments.py

@@ -81,7 +81,7 @@ def _specs(self, **kwargs):
 _arguments['module_type'] = Args(
     '-m', '--module-type',
     choices=spack.modules.module_types.keys(),
-    default=spack.modules.module_types.keys()[0],
+    default=list(spack.modules.module_types.keys())[0],
     help='type of module files [default: %(default)s]')
 
 _arguments['yes_to_all'] = Args(

lib/spack/spack/cmd/compiler.py

@@ -22,8 +22,11 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
+
 import argparse
 import sys
+from six import iteritems
 
 import llnl.util.tty as tty
 import spack.compilers
@@ -142,36 +145,36 @@ def compiler_info(args):
         tty.error("No compilers match spec %s" % cspec)
     else:
         for c in compilers:
-            print str(c.spec) + ":"
-            print "\tpaths:"
+            print(str(c.spec) + ":")
+            print("\tpaths:")
             for cpath in ['cc', 'cxx', 'f77', 'fc']:
-                print "\t\t%s = %s" % (cpath, getattr(c, cpath, None))
+                print("\t\t%s = %s" % (cpath, getattr(c, cpath, None)))
             if c.flags:
-                print "\tflags:"
-                for flag, flag_value in c.flags.iteritems():
-                    print "\t\t%s = %s" % (flag, flag_value)
+                print("\tflags:")
+                for flag, flag_value in iteritems(c.flags):
+                    print("\t\t%s = %s" % (flag, flag_value))
             if len(c.environment) != 0:
                 if len(c.environment['set']) != 0:
-                    print "\tenvironment:"
-                    print "\t    set:"
-                    for key, value in c.environment['set'].iteritems():
-                        print "\t        %s = %s" % (key, value)
+                    print("\tenvironment:")
+                    print("\t    set:")
+                    for key, value in iteritems(c.environment['set']):
+                        print("\t        %s = %s" % (key, value))
             if c.extra_rpaths:
-                print "\tExtra rpaths:"
+                print("\tExtra rpaths:")
                 for extra_rpath in c.extra_rpaths:
-                    print "\t\t%s" % extra_rpath
-            print "\tmodules  = %s" % c.modules
-            print "\toperating system  = %s" % c.operating_system
+                    print("\t\t%s" % extra_rpath)
+            print("\tmodules  = %s" % c.modules)
+            print("\toperating system  = %s" % c.operating_system)
 
 
 def compiler_list(args):
     tty.msg("Available compilers")
     index = index_by(spack.compilers.all_compilers(scope=args.scope),
                      lambda c: (c.spec.name, c.operating_system, c.target))
-    ordered_sections = sorted(index.items(), key=lambda (k, v): k)
+    ordered_sections = sorted(index.items(), key=lambda item: item[0])
     for i, (key, compilers) in enumerate(ordered_sections):
         if i >= 1:
-            print
+            print()
         name, os, target = key
         os_str = os
         if target:

lib/spack/spack/cmd/configure.py

@@ -22,7 +22,6 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-
 import argparse
 
 import llnl.util.tty as tty

lib/spack/spack/cmd/dependents.py

@@ -50,4 +50,4 @@ def dependents(parser, args):
     if deps:
         spack.cmd.display_specs(deps)
     else:
-        print "No dependents"
+        print("No dependents")

lib/spack/spack/cmd/env.py

@@ -22,8 +22,11 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
+
 import os
 import argparse
+
 import llnl.util.tty as tty
 import spack.cmd
 import spack.build_environment as build_env
@@ -64,7 +67,7 @@ def env(parser, args):
     if not cmd:
         # If no command act like the "env" command and print out env vars.
         for key, val in os.environ.items():
-            print "%s=%s" % (key, val)
+            print("%s=%s" % (key, val))
 
     else:
         # Otherwise execute the command with the new environment