Bug 784841 - Part 20: Fixes to support building external projects; r=…

…glandium

This is extremely hacky. It will likely need many refinements as we
figure out how we want external projects to be handled.

b2g/app.mozbuild

@@ -5,6 +5,8 @@
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = []
+    app_libxul_static_dirs = []
+
     include('/toolkit/toolkit.mozbuild')
 elif CONFIG['ENABLE_TESTS']:
     add_tier_dir('testharness', 'testing/mochitest')

browser/app.mozbuild

@@ -5,6 +5,7 @@
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = []
+    app_libxul_static_dirs = []
     include('/toolkit/toolkit.mozbuild')
 
 if CONFIG['MOZ_EXTENSIONS']:

configure.in

@@ -4296,6 +4296,12 @@ case "${target}" in
 
 esac
 
+MOZ_ARG_WITH_STRING(external-source-dir,
+[  --with-external-source-dir=dir
+                          External directory containing additional build files.],
+[ EXTERNAL_SOURCE_DIR=$withval])
+AC_SUBST(EXTERNAL_SOURCE_DIR)
+
 MOZ_ARG_ENABLE_STRING(application,
 [  --enable-application=APP
                           Options include:

mobile/android/app.mozbuild

@@ -5,6 +5,7 @@
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = ['mobile/android/components/build']
+    app_libxul_static_dirs = []
 
     include('/toolkit/toolkit.mozbuild')
 

mobile/xul/app.mozbuild

@@ -5,6 +5,7 @@
 
 if not CONFIG['LIBXUL_SDK']:
     app_libxul_dirs = ['mobile/xul/components/build']
+    app_libxul_static_dirs = []
     include('/toolkit/toolkit.mozbuild')
 
 elif CONFIG['ENABLE_TESTS']:

python/mozbuild/mozbuild/backend/base.py

@@ -35,6 +35,9 @@ def __init__(self, environment):
 
         self.environment = environment
 
+        self._environments = {}
+        self._environments[environment.topobjdir] = environment
+
         self._init()
 
     def _init():
@@ -43,6 +46,24 @@ def _init():
         This exists so child classes don't need to implement __init__.
         """
 
+    def get_environment(self, obj):
+        """Obtain the ConfigEnvironment for a specific object.
+
+        This is used to support external source directories which operate in
+        their own topobjdir and have their own ConfigEnvironment.
+
+        This is somewhat hacky and should be considered for rewrite if external
+        project integration is rewritten.
+        """
+        environment = self._environments.get(obj.topobjdir, None)
+        if not environment:
+            config_status = os.path.join(obj.topobjdir, 'config.status')
+
+            environment = ConfigEnvironment.from_config_status(config_status)
+            self._environments[obj.topobjdir] = environment
+
+        return environment
+
     def consume(self, objs):
         """Consume a stream of TreeMetadata instances.
 

python/mozbuild/mozbuild/backend/configenvironment.py

@@ -25,6 +25,39 @@ def shell_escape(s):
     return RE_SHELL_ESCAPE.sub(r'\\\1', str(s)).replace('$', '$$')
 
 
+class BuildConfig(object):
+    """Represents the output of configure."""
+
+    def __init__(self):
+        self.topsrcdir = None
+        self.topobjdir = None
+        self.defines = {}
+        self.non_global_defines = []
+        self.substs = {}
+        self.files = []
+
+    @staticmethod
+    def from_config_status(path):
+        """Create an instance from a config.status file."""
+
+        with open(path, 'rt') as fh:
+            source = fh.read()
+            code = compile(source, path, 'exec', dont_inherit=1)
+            g = {
+                '__builtins__': __builtins__,
+                '__file__': path,
+            }
+            l = {}
+            exec(code, g, l)
+
+            config = BuildConfig()
+
+            for name in l['__all__']:
+                setattr(config, name, l[name])
+
+            return config
+
+
 class ConfigEnvironment(object):
     """Perform actions associated with a configured but bare objdir.
 
@@ -80,6 +113,13 @@ def __init__(self, topsrcdir, topobjdir, defines=[], non_global_defines=[],
         self.substs['ALLDEFINES'] = '\n'.join(sorted(['#define %s %s' % (name,
             self.defines[name]) for name in global_defines]))
 
+    @staticmethod
+    def from_config_status(path):
+        config = BuildConfig.from_config_status(path)
+
+        return ConfigEnvironment(config.topsrcdir, config.topobjdir,
+            config.defines, config.non_global_defines, config.substs)
+
     def get_relative_srcdir(self, file):
         '''Returns the relative source directory for the given file, always
         using / as a path separator.

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -59,9 +59,10 @@ class BackendMakeFile(object):
     of date during recursion.
     """
 
-    def __init__(self, srcdir, objdir):
+    def __init__(self, srcdir, objdir, environment):
         self.srcdir = srcdir
         self.objdir = objdir
+        self.environment = environment
         self.path = os.path.join(objdir, 'backend.mk')
 
         # Filenames that influenced the content of this file.
@@ -130,7 +131,7 @@ def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
 
         backend_file = self._backend_files.get(obj.srcdir,
-            BackendMakeFile(obj.srcdir, obj.objdir))
+            BackendMakeFile(obj.srcdir, obj.objdir, self.get_environment(obj)))
 
         # Define the paths that will trigger a backend rebuild. We always
         # add autoconf.mk because that is proxy for CONFIG. We can't use
@@ -144,7 +145,7 @@ def consume_object(self, obj):
         elif isinstance(obj, ConfigFileSubstitution):
             backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath)
 
-            self.environment.create_config_file(obj.output_path)
+            backend_file.environment.create_config_file(obj.output_path)
 
         self._backend_files[obj.srcdir] = backend_file
 
@@ -163,7 +164,7 @@ def consume_finished(self):
             out_path = os.path.join(bf.objdir, 'Makefile')
             self.log(logging.DEBUG, 'create_makefile', {'path': out_path},
                 'Generating makefile: {path}')
-            self.environment.create_config_file(out_path)
+            bf.environment.create_config_file(out_path)
 
             bf.write('SUBSTITUTE_FILES += Makefile\n')
             bf.close()

python/mozbuild/mozbuild/frontend/reader.py

@@ -56,10 +56,42 @@
     text_type = str
     type_type = type
 
+
 def log(logger, level, action, params, formatter):
     logger.log(level, formatter, extra={'action': action, 'params': params})
 
 
+def is_read_allowed(path, config):
+    """Whether we are allowed to load a mozbuild file at the specified path.
+
+    This is used as cheap security to ensure the build is isolated to known
+    source directories.
+
+    We are allowed to read from the main source directory and any defined
+    external source directories. The latter is to allow 3rd party applications
+    to hook into our build system.
+    """
+    assert os.path.isabs(path)
+    assert os.path.isabs(config.topsrcdir)
+
+    path = os.path.normpath(path)
+    topsrcdir = os.path.normpath(config.topsrcdir)
+
+    if path.startswith(topsrcdir):
+        return True
+
+    external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+    for external in external_dirs:
+        if not os.path.isabs(external):
+            external = os.path.join(config.topsrcdir, external)
+        external = os.path.normpath(external)
+
+        if path.startswith(external):
+            return True
+
+    return False
+
+
 class SandboxCalledError(SandboxError):
     """Represents an error resulting from calling the error() function."""
 
@@ -87,19 +119,38 @@ def __init__(self, config, path):
 
         topobjdir = os.path.abspath(config.topobjdir)
 
-        # This may not always hold true. If we ever have autogenerated mozbuild
-        # files in topobjdir, we'll need to change this.
-        assert os.path.normpath(path).startswith(os.path.normpath(config.topsrcdir))
-        assert not os.path.normpath(path).startswith(os.path.normpath(topobjdir))
+        topsrcdir = config.topsrcdir
+        if not path.startswith(topsrcdir):
+            external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+            for external in external_dirs:
+                if not os.path.isabs(external):
+                    external = os.path.join(config.topsrcdir, external)
+
+                external = os.path.normpath(external)
+
+                if not path.startswith(external):
+                    continue
 
-        relpath = os.path.relpath(path, config.topsrcdir).replace(os.sep, '/')
+                topsrcdir = external
+
+                # This is really hacky and should be replaced with something
+                # more robust. We assume that if an external source directory
+                # is in play that the main build system is built in a
+                # subdirectory of its topobjdir. Therefore, the topobjdir of
+                # the external source directory is the parent of our topobjdir.
+                topobjdir = os.path.dirname(topobjdir)
+                break
+
+        self.topsrcdir = topsrcdir
+
+        relpath = os.path.relpath(path, topsrcdir).replace(os.sep, '/')
         reldir = os.path.dirname(relpath)
 
         with self._globals.allow_all_writes() as d:
-            d['TOPSRCDIR'] = config.topsrcdir
+            d['TOPSRCDIR'] = topsrcdir
             d['TOPOBJDIR'] = topobjdir
             d['RELATIVEDIR'] = reldir
-            d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
+            d['SRCDIR'] = os.path.join(topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
             d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
 
             # config.status does not yet use unicode. However, mozbuild expects
@@ -118,8 +169,6 @@ def __init__(self, config, path):
             for name, func in FUNCTIONS.items():
                 d[name] = getattr(self, func[0])
 
-        self._normalized_topsrcdir = os.path.normpath(config.topsrcdir)
-
     def exec_file(self, path, filesystem_absolute=False):
         """Override exec_file to normalize paths and restrict file loading.
 
@@ -136,7 +185,7 @@ def exec_file(self, path, filesystem_absolute=False):
         """
         if os.path.isabs(path):
             if not filesystem_absolute:
-                path = os.path.normpath(os.path.join(self.config.topsrcdir,
+                path = os.path.normpath(os.path.join(self.topsrcdir,
                     path[1:]))
 
         else:
@@ -146,12 +195,12 @@ def exec_file(self, path, filesystem_absolute=False):
                     path))
             else:
                 path = os.path.normpath(os.path.join(
-                    self.config.topsrcdir, path))
+                    self.topsrcdir, path))
 
         # realpath() is needed for true security. But, this isn't for security
         # protection, so it is omitted.
         normalized_path = os.path.normpath(path)
-        if not normalized_path.startswith(self._normalized_topsrcdir):
+        if not is_read_allowed(normalized_path, self.config):
             raise SandboxLoadError(list(self._execution_stack),
                 sys.exc_info()[2], illegal_path=path)
 
@@ -484,7 +533,6 @@ def __init__(self, config):
 
         self._log = logging.getLogger(__name__)
         self._read_files = set()
-        self._normalized_topsrcdir = os.path.normpath(config.topsrcdir)
         self._execution_stack = []
 
     def read_topsrcdir(self):
@@ -615,9 +663,9 @@ def _read_mozbuild(self, path, read_tiers, filesystem_absolute, descend):
             # that's not our problem. We're not a hosted application: we don't
             # need to worry about security too much.
             child_path = os.path.normpath(child_path)
-            if not child_path.startswith(self._normalized_topsrcdir):
+            if not is_read_allowed(child_path, self.config):
                 raise SandboxValidationError(
-                    'Attempting to process file outside of topsrcdir: %s' %
+                    'Attempting to process file outside of allowed paths: %s' %
                         child_path)
 
             if not descend:

toolkit/toolkit.mozbuild

@@ -209,6 +209,7 @@ if CONFIG['MOZ_GIO_COMPONENT']:
 # Applications can cheat and ask for code to be
 # built before libxul so it can be linked into libxul.
 add_tier_dir('platform', app_libxul_dirs)
+add_tier_dir('platform', app_libxul_static_dirs, static=True)
 
 add_tier_dir('platform', 'toolkit/library')
 add_tier_dir('platform', 'xpcom/stub')

xulrunner/app.mozbuild

@@ -4,6 +4,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 app_libxul_dirs = []
+app_libxul_static_dirs = []
 include('/toolkit/toolkit.mozbuild')
 
 if CONFIG['MOZ_EXTENSIONS']: