Bug 1651287 - Add a pre-compile pseudo-tier. r=firefox-build-system-r…

…eviewers,rstewart

Generated files marked as "required during compile" are limited to a
number of extensions, most of which are source file types, so they get
generated when compiling the corresponding object file. Other types,
though, are currently handled via EXTRA_DEPS, which is clunky.

As of now, these other types are limited to link-related items, but more
would be useful (e.g. .inc files, which are only included in one
location).

This however works against the static analysis tasks (coverity and
clang-tidy), which currently rely on everything they need being created
via the export tier. That excludes using EXTRA_DEPS-based hacks.

We create a "pre-compile" tier, that is not used during a normal build,
but can be invoked manually, which the static analysis tasks will do.

Differential Revision: https://phabricator.services.mozilla.com/D83035

config/baseconfig.mk

@@ -38,7 +38,7 @@ ifndef INCLUDED_AUTOCONF_MK
 default::
 else
 # All possible tiers
-ALL_TIERS := artifact win32-artifact android-fat-aar-artifact pre-export export rust compile misc libs android-stage-package android-archive-geckoview tools check
+ALL_TIERS := artifact win32-artifact android-fat-aar-artifact pre-export export pre-compile rust compile misc libs android-stage-package android-archive-geckoview tools check
 
 # All tiers that may be used manually via `mach build $tier`
 RUNNABLE_TIERS := $(ALL_TIERS)
@@ -57,7 +57,7 @@ RUNNABLE_TIERS := $(filter-out android-archive-geckoview,$(RUNNABLE_TIERS))
 endif
 
 # All tiers that run automatically on `mach build`
-TIERS := $(filter-out check,$(RUNNABLE_TIERS))
+TIERS := $(filter-out pre-compile check,$(RUNNABLE_TIERS))
 ifndef COMPILE_ENVIRONMENT
 TIERS := $(filter-out rust compile,$(TIERS))
 endif

python/mozbuild/mozbuild/backend/make.py

@@ -80,8 +80,9 @@ def _format_statements_for_generated_file(self, obj, tier,
                 if tier and not needs_AB_rCD:
                     # Android localized resources have special Makefile
                     # handling.
-                    ret.append('%s%s: %s' % (
-                        tier, ':' if tier in ('export', 'libs', 'misc') else '', stub_file))
+                    double_colon_tiers = ('export', 'pre-compile', 'libs', 'misc')
+                    ret.append('%s%s %s' % (
+                        tier, '::' if tier in double_colon_tiers else ':', stub_file))
             for output in outputs:
                 ret.append('%s: %s ;' % (output, stub_file))
                 ret.append('GARBAGE += %s' % output)

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -379,6 +379,7 @@ def _init(self):
         self._rust_targets = set()
         self._rust_lib_targets = set()
         self._gkrust_target = None
+        self._pre_compile = set()
 
         self._no_skip = {
             'export': set(),
@@ -546,18 +547,20 @@ def consume_object(self, obj):
             if obj.required_before_compile:
                 tier = 'export'
             elif obj.required_during_compile:
-                tier = None
+                tier = 'pre-compile'
             else:
                 tier = 'misc'
-            if tier:
-                relobjdir = mozpath.relpath(obj.objdir, self.environment.topobjdir)
+            relobjdir = mozpath.relpath(obj.objdir, self.environment.topobjdir)
+            if tier == 'pre-compile':
+                self._pre_compile.add(relobjdir)
+            else:
                 self._no_skip[tier].add(relobjdir)
             backend_file.write_once('include $(topsrcdir)/config/AB_rCD.mk\n')
             relobjdir = mozpath.relpath(obj.objdir, backend_file.objdir)
             # For generated files that we handle in the top-level backend file,
             # we want to have a `directory/tier` target depending on the file.
             # For the others, we want a `tier` target.
-            if tier and relobjdir:
+            if tier != 'pre-compile' and relobjdir:
                 tier = '%s/%s' % (relobjdir, tier)
             for stmt in self._format_statements_for_generated_file(
                     obj, tier, extra_dependencies='backend.mk' if obj.flags else ''):
@@ -741,6 +744,9 @@ def tools_filter(current, subdirs):
             if main:
                 rule.add_dependencies('%s/%s' % (d, tier) for d in sorted(main))
 
+        rule = root_deps_mk.create_rule(['recurse_pre-compile'])
+        rule.add_dependencies('%s/pre-compile' % d for d in sorted(self._pre_compile))
+
         all_compile_deps = six.moves.reduce(
             lambda x, y: x | y,
             self._compile_graph.values()) if self._compile_graph else set()

python/mozbuild/mozbuild/code-analysis/mach_commands.py

@@ -1948,9 +1948,14 @@ def on_line(line):
 
         # Then build the rest of the build dependencies by running the full
         # export target, because we can't do anything better.
-        return builder._run_make(directory=self.topobjdir, target='export',
-                                 line_handler=None, silent=not verbose,
-                                 num_jobs=jobs)
+        for target in ('export', 'pre-compile'):
+            rc = builder._run_make(directory=self.topobjdir, target=target,
+                                   line_handler=None, silent=not verbose,
+                                   num_jobs=jobs)
+            if rc != 0:
+                return rc
+
+        return 0
 
     def _set_clang_tools_paths(self):
         rc, config, _ = self._get_config_environment()

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -410,6 +410,7 @@ def test_generated_files(self):
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
+            'pre-compile:: $(MDDEPDIR)/bar.c.stub',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
@@ -443,6 +444,7 @@ def test_generated_files_force(self):
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
+            'pre-compile:: $(MDDEPDIR)/bar.c.stub',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
@@ -452,6 +454,7 @@ def test_generated_files_force(self):
             '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
+            'pre-compile:: $(MDDEPDIR)/foo.c.stub',
             'foo.c: $(MDDEPDIR)/foo.c.stub ;',
             'GARBAGE += foo.c',
             'GARBAGE += $(MDDEPDIR)/foo.c.stub',