Bug 1250297 - Make python configure output config.status instead of o…

…ld-configure doing it. r=gps

The nice side effect is that now we can have actual dicts for defines
and substs from the start, which simplifies so things, although it
requires adjustments to some unit tests.

CLOBBER

@@ -22,5 +22,5 @@
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Bug 1238079 seems to have needed a clobber for Mac
+Bug 1250297 - Doesn't actually need a clobber, but updating the tree to an older changeset does.
 

build/autoconf/config.status.m4

@@ -83,11 +83,9 @@ dnl Replace AC_OUTPUT to create and call a python config.status
 define([MOZ_CREATE_CONFIG_STATUS],
 [dnl Top source directory in Windows format (as opposed to msys format).
 WIN_TOP_SRC=
-encoding=utf-8
 case "$host_os" in
 mingw*)
     WIN_TOP_SRC=`cd $srcdir; pwd -W`
-    encoding=mbcs
     ;;
 esac
 AC_SUBST(WIN_TOP_SRC)
@@ -101,7 +99,7 @@ trap '' 1 2 15
 AC_CACHE_SAVE
 
 trap 'rm -f $CONFIG_STATUS conftest*; exit 1' 1 2 15
-: ${CONFIG_STATUS=./config.status}
+: ${CONFIG_STATUS=./config.data}
 
 dnl We're going to need [ ] for python syntax.
 changequote(<<<, >>>)dnl
@@ -110,21 +108,6 @@ echo creating $CONFIG_STATUS
 extra_python_path=${COMM_BUILD:+"'mozilla', "}
 
 cat > $CONFIG_STATUS <<EOF
-#!${PYTHON}
-# coding=$encoding
-
-import os
-import types
-dnl topsrcdir is the top source directory in native form, as opposed to a
-dnl form suitable for make.
-topsrcdir = '''${WIN_TOP_SRC:-$srcdir}'''
-if not os.path.isabs(topsrcdir):
-    rel = os.path.join(os.path.dirname(<<<__file__>>>), topsrcdir)
-    topsrcdir = os.path.abspath(rel)
-topsrcdir = os.path.normpath(topsrcdir)
-
-topobjdir = os.path.abspath(os.path.dirname(<<<__file__>>>))
-
 def unique_list(l):
     result = []
     for i in l:
@@ -135,7 +118,7 @@ def unique_list(l):
 dnl All defines and substs are stored with an additional space at the beginning
 dnl and at the end of the string, to avoid any problem with values starting or
 dnl ending with quotes.
-defines = [(name[1:-1], value[1:-1]) for name, value in [
+defines = [
 EOF
 
 dnl confdefs.pytmp contains AC_DEFINEs, in the expected format, but
@@ -144,9 +127,9 @@ sed 's/$/,/' confdefs.pytmp >> $CONFIG_STATUS
 rm confdefs.pytmp confdefs.h
 
 cat >> $CONFIG_STATUS <<\EOF
-] ]
+]
 
-substs = [(name[1:-1], value[1:-1] if isinstance(value, types.StringTypes) else value) for name, value in [
+substs = [
 EOF
 
 dnl The MOZ_DIVERSION_SUBST output diversion contains AC_SUBSTs, in the
@@ -162,7 +145,7 @@ for ac_subst_arg in $_subconfigure_ac_subst_args; do
 done
 
 cat >> $CONFIG_STATUS <<\EOF
-] ]
+]
 
 dnl List of AC_DEFINEs that aren't to be exposed in ALLDEFINES
 non_global_defines = [
@@ -176,39 +159,9 @@ fi
 
 cat >> $CONFIG_STATUS <<EOF
 ]
-
-__all__ = ['topobjdir', 'topsrcdir', 'defines', 'non_global_defines', 'substs']
-EOF
-
-# We don't want js/src/config.status to do anything in gecko builds.
-if test -z "$BUILDING_JS" -o -n "$JS_STANDALONE"; then
-
-    cat >> $CONFIG_STATUS <<EOF
-dnl Do the actual work
-if __name__ == '__main__':
-    args = dict([(name, globals()[name]) for name in __all__])
-    from mozbuild.config_status import config_status
-    config_status(**args)
 EOF
 
-fi
-
 changequote([, ])
-
-chmod +x $CONFIG_STATUS
-])
-
-define([MOZ_RUN_CONFIG_STATUS],
-[
-
-MOZ_RUN_ALL_SUBCONFIGURES()
-
-rm -fr confdefs* $ac_clean_files
-dnl Execute config.status, unless --no-create was passed to configure.
-if test "$no_create" != yes && ! ${PYTHON} $CONFIG_STATUS; then
-    trap '' EXIT
-    exit 1
-fi
 ])
 
 define([m4_fatal],[

configure.py

@@ -4,14 +4,18 @@
 
 from __future__ import print_function, unicode_literals
 
+import codecs
 import glob
 import itertools
+import json
 import os
 import subprocess
 import sys
 import re
+import types
 
-base_dir = os.path.dirname(__file__)
+
+base_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.append(os.path.join(base_dir, 'python', 'which'))
 sys.path.append(os.path.join(base_dir, 'python', 'mozbuild'))
 from which import which, WhichError
@@ -116,8 +120,70 @@ def main(args):
         print(e.message, file=sys.stderr)
         return 1
 
-    return subprocess.call([shell, old_configure] + args)
-
+    ret = subprocess.call([shell, old_configure] + args)
+    # We don't want to create and run config.status if --help was one of the
+    # command line arguments.
+    if ret or '--help' in args:
+        return ret
+
+    raw_config = {}
+    encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
+    with codecs.open('config.data', 'r', encoding) as fh:
+        code = compile(fh.read(), 'config.data', 'exec')
+        # Every variation of the exec() function I tried led to:
+        # SyntaxError: unqualified exec is not allowed in function 'main' it
+        # contains a nested function with free variables
+        exec code in raw_config
+    # If the code execution above fails, we want to keep the file around for
+    # debugging.
+    os.remove('config.data')
+
+    # Sanitize config data
+    config = {}
+    substs = config['substs'] = {
+        k[1:-1]: v[1:-1] if isinstance(v, types.StringTypes) else v
+        for k, v in raw_config['substs']
+    }
+    config['defines'] = {
+        k[1:-1]: v[1:-1]
+        for k, v in raw_config['defines']
+    }
+    config['non_global_defines'] = raw_config['non_global_defines']
+    config['topsrcdir'] = base_dir
+    config['topobjdir'] = os.path.abspath(os.getcwd())
+
+    # Create config.status. Eventually, we'll want to just do the work it does
+    # here, when we're able to skip configure tests/use cached results/not rely
+    # on autoconf.
+    print("Creating config.status", file=sys.stderr)
+    with codecs.open('config.status', 'w', encoding) as fh:
+        fh.write('#!%s\n' % config['substs']['PYTHON'])
+        fh.write('# coding=%s\n' % encoding)
+        for k, v in config.iteritems():
+            fh.write('%s = ' % k)
+            json.dump(v, fh, sort_keys=True, indent=4, ensure_ascii=False)
+            fh.write('\n')
+        fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
+                 "'non_global_defines', 'substs']")
+
+        if not substs.get('BUILDING_JS') or substs.get('JS_STANDALONE'):
+            fh.write('''
+if __name__ == '__main__':
+    args = dict([(name, globals()[name]) for name in __all__])
+    from mozbuild.config_status import config_status
+    config_status(**args)
+''')
+
+    # Other things than us are going to run this file, so we need to give it
+    # executable permissions.
+    os.chmod('config.status', 0755)
+    if not substs.get('BUILDING_JS') or substs.get('JS_STANDALONE'):
+        if not substs.get('JS_STANDALONE'):
+            os.environ['WRITE_MOZINFO'] = '1'
+        # Until we have access to the virtualenv from this script, execute
+        # config.status externally, with the virtualenv python.
+        return subprocess.call([config['substs']['PYTHON'], 'config.status'])
+    return 0
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))

js/src/old-configure.in

@@ -168,6 +168,7 @@ else
 fi
 AC_SUBST(JS_STANDALONE)
 BUILDING_JS=1
+AC_SUBST(BUILDING_JS)
 AC_SUBST(autoconfmk)
 
 MOZ_ARG_WITH_STRING(gonk,
@@ -3667,5 +3668,7 @@ dnl ========================================================
 MOZ_CREATE_CONFIG_STATUS()
 
 if test "$JS_STANDALONE"; then
-  MOZ_RUN_CONFIG_STATUS()
+  MOZ_RUN_ALL_SUBCONFIGURES()
 fi
+
+rm -fr confdefs* $ac_clean_files

old-configure.in

@@ -9186,11 +9186,11 @@ ac_configure_args="$_SUBDIR_CONFIG_ARGS"
 
 fi # COMPILE_ENVIRONMENT
 
-export WRITE_MOZINFO=1
 dnl we need to run config.status after js/src subconfigure because we're
 dnl traversing its moz.build and we need its config.status for that.
 dnl However, writing our own config.status needs to happen before
 dnl subconfigures because the setup surrounding subconfigures alters
 dnl many AC_SUBSTed variables.
-MOZ_RUN_CONFIG_STATUS()
-unset WRITE_MOZINFO
+MOZ_RUN_ALL_SUBCONFIGURES()
+
+rm -fr confdefs* $ac_clean_files

python/mozbuild/mozbuild/backend/configenvironment.py

@@ -78,12 +78,11 @@ class ConfigEnvironment(object):
     Creating a ConfigEnvironment requires a few arguments:
       - topsrcdir and topobjdir are, respectively, the top source and
         the top object directory.
-      - defines is a list of (name, value) tuples. In autoconf, these are
-        set with AC_DEFINE and AC_DEFINE_UNQUOTED
+      - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in
+        autoconf.
       - non_global_defines are a list of names appearing in defines above
         that are not meant to be exported in ACDEFINES (see below)
-      - substs is a list of (name, value) tuples. In autoconf, these are
-        set with AC_SUBST.
+      - substs is a dict filled from AC_SUBST in autoconf.
 
     ConfigEnvironment automatically defines one additional substs variable
     from all the defines not appearing in non_global_defines:
@@ -106,15 +105,15 @@ class ConfigEnvironment(object):
     path or a path relative to the topobjdir.
     """
 
-    def __init__(self, topsrcdir, topobjdir, defines=[], non_global_defines=[],
-        substs=[], source=None):
+    def __init__(self, topsrcdir, topobjdir, defines=None,
+        non_global_defines=None, substs=None, source=None):
 
         if not source:
             source = mozpath.join(topobjdir, 'config.status')
         self.source = source
-        self.defines = ReadOnlyDict(defines)
-        self.non_global_defines = non_global_defines
-        self.substs = dict(substs)
+        self.defines = ReadOnlyDict(defines or {})
+        self.non_global_defines = non_global_defines or []
+        self.substs = dict(substs or {})
         self.topsrcdir = mozpath.abspath(topsrcdir)
         self.topobjdir = mozpath.abspath(topobjdir)
         self.lib_prefix = self.substs.get('LIB_PREFIX', '')
@@ -129,10 +128,11 @@ def __init__(self, topsrcdir, topobjdir, defines=[], non_global_defines=[],
             self.import_prefix = self.dll_prefix
             self.import_suffix = self.dll_suffix
 
-        global_defines = [name for name, value in defines
-            if not name in non_global_defines]
+        global_defines = [name for name in self.defines
+            if not name in self.non_global_defines]
         self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
-            shell_quote(self.defines[name]).replace('$', '$$')) for name in global_defines])
+            shell_quote(self.defines[name]).replace('$', '$$'))
+            for name in sorted(global_defines)])
         def serialize(obj):
             if isinstance(obj, StringTypes):
                 return obj

python/mozbuild/mozbuild/config_status.py

@@ -62,8 +62,8 @@
 '''.strip()
 
 
-def config_status(topobjdir='.', topsrcdir='.',
-        defines=[], non_global_defines=[], substs=[], source=None):
+def config_status(topobjdir='.', topsrcdir='.', defines=None,
+                  non_global_defines=None, substs=None, source=None):
     '''Main function, providing config.status functionality.
 
     Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
@@ -95,13 +95,7 @@ def config_status(topobjdir='.', topsrcdir='.',
             '%s' % topsrcdir)
 
     default_backends = ['RecursiveMake']
-    # We have a chicken/egg problem, where we only have a dict for substs after
-    # creating the ConfigEnvironment, which requires argument parsing to have
-    # occurred.
-    for name, value in substs:
-        if name == 'BUILD_BACKENDS':
-            default_backends = value
-            break
+    default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake'])
 
     parser = ArgumentParser()
     parser.add_argument('--recheck', dest='recheck', action='store_true',