Bug 1563403 - Use 3-tier PGO for local MOZ_PGO=1 builds; r=firefox-bu…

…ild-system-reviewers,dmajor,chmanchester

Local PGO builds now use 3-tier machinery under the hood. Instead of a
single object directory that gets cleaned in between the instrumented
and profile-use builds, now the instrumented build uses a separate
'${objdir}/instrumented' directory. This makes it easier to handle
within mach since we can drive the two builds with environment variables
and keep all build artifacts separate, without needing to do manual
cleanup in between.

Differential Revision: https://phabricator.services.mozilla.com/D50098

Makefile.in

@@ -115,11 +115,8 @@ install-test-files:
 
 include $(topsrcdir)/build/moz-automation.mk
 
-# dist and _tests should be purged during cleaning. However, we don't want them
-# purged during PGO builds because they contain some auto-generated files.
-ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
+# dist and _tests should be purged during cleaning.
 GARBAGE_DIRS += dist _tests
-endif
 
 # Dummy rule for the cases below where we don't depend on dist/include
 recurse_pre-export::
@@ -185,37 +182,6 @@ endif
 default all::
 	$(call BUILDSTATUS,TIERS $(TIERS) $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))
 
-# PGO build target.
-profiledbuild::
-	$(call BUILDSTATUS,TIERS pgo_profile_generate pgo_package pgo_profile pgo_clobber pgo_profile_use)
-	$(call BUILDSTATUS,TIER_START pgo_profile_generate)
-	$(MAKE) default MOZ_PROFILE_GENERATE=1 MOZ_LTO=
-	$(call BUILDSTATUS,TIER_FINISH pgo_profile_generate)
-	$(call BUILDSTATUS,TIER_START pgo_package)
-	$(MAKE) package
-	rm -f jarlog/en-US.log
-	$(call BUILDSTATUS,TIER_FINISH pgo_package)
-	$(call BUILDSTATUS,TIER_START pgo_profile)
-	JARLOG_FILE=jarlog/en-US.log $(PYTHON) $(topsrcdir)/build/pgo/profileserver.py
-	$(call BUILDSTATUS,TIER_FINISH pgo_profile)
-	$(call BUILDSTATUS,TIER_START pgo_clobber)
-	$(MAKE) maybe_clobber_profiledbuild
-	$(call BUILDSTATUS,TIER_FINISH pgo_clobber)
-	$(call BUILDSTATUS,TIER_START pgo_profile_use)
-	$(MAKE) default MOZ_PROFILE_USE=1 MOZ_1TIER_PGO=1
-	$(call BUILDSTATUS,TIER_FINISH pgo_profile_use)
-
-# Change default target to PGO build if PGO is enabled.
-ifdef MOZ_PGO
-ifdef COMPILE_ENVIRONMENT
-# If one of these is already set in addition to PGO we are doing a single phase
-# of PGO in isolation, so don't override the default target.
-ifeq (,$(MOZ_PROFILE_GENERATE)$(MOZ_PROFILE_USE))
-OVERRIDE_DEFAULT_GOAL := profiledbuild
-endif
-endif
-endif
-
 include $(topsrcdir)/config/rules.mk
 
 ifdef SCCACHE_VERBOSE_STATS
@@ -281,19 +247,6 @@ update-packaging:
 package-generated-sources:
 	$(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)')
 
-# PGO support, but we can't do this test in client.mk
-# No point in clobbering if PGO has been explicitly disabled.
-ifdef NO_PROFILE_GUIDED_OPTIMIZE
-maybe_clobber_profiledbuild:
-else
-maybe_clobber_profiledbuild: clean
-ifneq (,$(findstring clang,$(CC_TYPE)))
-	$(LLVM_PROFDATA) merge -o $(DEPTH)/merged.profdata $(DEPTH)/*.profraw
-endif
-endif # NO_PROFILE_GUIDED_OPTIMIZE
-
-.PHONY: maybe_clobber_profiledbuild
-
 ifdef JS_STANDALONE
 # Delegate js-specific rules to js
 check-%:

build/docs/pgo.rst

@@ -23,17 +23,6 @@ Then::
 
 This is roughly equivalent to::
 
-#. Perform a build with *MOZ_PROFILE_GENERATE=1*
-#. Performing a run of the instrumented binaries
-#. $ make maybe_clobber_profiledbuild
-#. Perform a build with *MOZ_PROFILE_USE=1*
-
-Differences between toolchains
-==============================
-
-There are some implementation differences depending on the compiler
-toolchain being used.
-
-The *maybe_clobber_profiledbuild* step gets its name because of a
-difference. On Windows, this step merely moves some *.pgc* files around.
-Using GCC or Clang, it is equivalent to a *make clean*.
+#. Perform a build with *--enable-profile-generate* in $topobjdir/instrumented
+#. Perform a run of the instrumented binaries with build/pgo/profileserver.py
+#. Perform a build with *--enable-profile-use* in $topobjdir

build/moz.configure/lto-pgo.configure

@@ -11,6 +11,7 @@ llvm_profdata = check_prog('LLVM_PROFDATA', ['llvm-profdata'],
                            paths=toolchain_search_path)
 
 js_option('--enable-profile-generate',
+          env='MOZ_PROFILE_GENERATE',
           nargs='?',
           choices=('cross',),
           help='Build a PGO instrumented binary')
@@ -25,6 +26,7 @@ set_define('MOZ_PROFILE_GENERATE',
            depends_if('--enable-profile-generate')(lambda _: True))
 
 js_option('--enable-profile-use',
+          env='MOZ_PROFILE_USE',
           nargs='?',
           choices=('cross',),
           help='Use a generated profile during the build')
@@ -51,7 +53,7 @@ def pgo_profile_path(path, pgo_use, profdata, build_env):
         topobjdir = topobjdir[:-7]
 
     if not path:
-        return os.path.join(topobjdir, 'merged.profdata')
+        return os.path.join(topobjdir,  'instrumented', 'merged.profdata')
     if path and not pgo_use:
         die('Pass --enable-profile-use to use --with-pgo-profile-path.')
     if path and not profdata:

build/pgo/profileserver.py

@@ -80,6 +80,10 @@ def get_crashreports(directory, name=None):
                        port=PORT,
                        options='primary,privileged')
 
+    old_profraw_files = glob.glob('*.profraw')
+    for f in old_profraw_files:
+        os.remove(f)
+
     with TemporaryDirectory() as profilePath:
         # TODO: refactor this into mozprofile
         profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
@@ -151,6 +155,8 @@ def get_crashreports(directory, name=None):
         if jarlog:
             env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
             print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
+            if os.path.exists(jarlog):
+                os.remove(jarlog)
 
         if 'UPLOAD_PATH' in env:
             process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-2.log')

python/mozbuild/mozbuild/build_commands.py

@@ -6,6 +6,7 @@
 
 import argparse
 import os
+import subprocess
 
 from mach.decorators import (
     CommandArgument,
@@ -14,6 +15,9 @@
 )
 
 from mozbuild.base import MachCommandBase
+from mozbuild.util import ensure_subprocess_env
+from mozbuild.mozconfig import MozconfigLoader
+import mozpack.path as mozpath
 
 from mozbuild.backend import (
     backends,
@@ -73,6 +77,54 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
 
         self.log_manager.enable_all_structured_loggers()
 
+        loader = MozconfigLoader(self.topsrcdir)
+        mozconfig = loader.read_mozconfig(loader.AUTODETECT)
+        doing_pgo = 'MOZ_PGO=1' in mozconfig['configure_args']
+        append_env = None
+
+        if doing_pgo:
+            if what:
+                raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' %
+                                what)
+            instr = self._spawn(BuildDriver)
+            orig_topobjdir = instr._topobjdir
+            instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented')
+
+            append_env = {'MOZ_PROFILE_GENERATE': '1'}
+            status = instr.build(
+                what=what,
+                disable_extra_make_dependencies=disable_extra_make_dependencies,
+                jobs=jobs,
+                directory=directory,
+                verbose=verbose,
+                keep_going=keep_going,
+                mach_context=self._mach_context,
+                append_env=append_env)
+            if status != 0:
+                return status
+
+            # Packaging the instrumented build is required to get the jarlog
+            # data.
+            status = instr._run_make(
+                directory=".", target='package',
+                silent=not verbose, ensure_exit_code=False,
+                append_env=append_env)
+            if status != 0:
+                return status
+
+            pgo_env = os.environ.copy()
+            pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA')
+            pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log')
+            pgo_cmd = [
+                instr.virtualenv_manager.python_path,
+                mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'),
+            ]
+            subprocess.check_call(pgo_cmd, cwd=instr.topobjdir,
+                                  env=ensure_subprocess_env(pgo_env))
+
+            # Set the default build to MOZ_PROFILE_USE
+            append_env = {'MOZ_PROFILE_USE': '1'}
+
         driver = self._spawn(BuildDriver)
         return driver.build(
             what=what,
@@ -81,7 +133,8 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
             directory=directory,
             verbose=verbose,
             keep_going=keep_going,
-            mach_context=self._mach_context)
+            mach_context=self._mach_context,
+            append_env=append_env)
 
     @Command('configure', category='build',
              description='Configure the tree (run configure and config.status).')

python/mozbuild/mozbuild/controller/building.py

@@ -996,7 +996,8 @@ def __init__(self, *args, **kwargs):
         self.mach_context = None
 
     def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
-              directory=None, verbose=False, keep_going=False, mach_context=None):
+              directory=None, verbose=False, keep_going=False, mach_context=None,
+              append_env=None):
         """Invoke the build backend.
 
         ``what`` defines the thing to build. If not defined, the default
@@ -1065,7 +1066,8 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
                     print(" Config object not found by mach.")
 
                 config_rc = self.configure(buildstatus_messages=True,
-                                           line_handler=output.on_line)
+                                           line_handler=output.on_line,
+                                           append_env=append_env)
 
                 if config_rc != 0:
                     return config_rc
@@ -1158,12 +1160,13 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
                     # could potentially be fixed if the build monitor were more
                     # intelligent about encountering undefined state.
                     no_build_status = b'1' if make_dir is not None else b''
+                    tgt_env = dict(append_env or {})
+                    tgt_env['NO_BUILDSTATUS_MESSAGES'] = no_build_status
                     status = self._run_make(
                         directory=make_dir, target=make_target,
                         line_handler=output.on_line, log=False, print_directory=False,
                         ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
-                        append_env={
-                            b'NO_BUILDSTATUS_MESSAGES': no_build_status},
+                        append_env=tgt_env,
                         keep_going=keep_going)
 
                     if status != 0:
@@ -1175,7 +1178,8 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
                 status = self._run_client_mk(line_handler=output.on_line,
                                              jobs=jobs,
                                              verbose=verbose,
-                                             keep_going=keep_going)
+                                             keep_going=keep_going,
+                                             append_env=append_env)
 
             self.log(logging.WARNING, 'warning_summary',
                      {'count': len(monitor.warnings_database)},
@@ -1338,7 +1342,7 @@ def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
         return status
 
     def configure(self, options=None, buildstatus_messages=False,
-                  line_handler=None):
+                  line_handler=None, append_env=None):
         # Disable indexing in objdir because it is not necessary and can slow
         # down builds.
         mkdir(self.topobjdir, not_indexed=True)
@@ -1350,12 +1354,13 @@ def on_line(line):
         line_handler = line_handler or on_line
 
         options = ' '.join(shell_quote(o) for o in options or ())
-        append_env = {b'CONFIGURE_ARGS': options.encode('utf-8')}
+        append_env = dict(append_env or {})
+        append_env['CONFIGURE_ARGS'] = options
 
         # Only print build status messages when we have an active
         # monitor.
         if not buildstatus_messages:
-            append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1'
+            append_env['NO_BUILDSTATUS_MESSAGES'] = b'1'
         status = self._run_client_mk(target='configure',
                                      line_handler=line_handler,
                                      append_env=append_env)

python/mozbuild/mozbuild/mozconfig.py

@@ -349,6 +349,12 @@ def filt(x, y): return {k: v for k, v in x.items() if k not in y}
 
             if name == 'MOZ_OBJDIR':
                 result['topobjdir'] = value
+                if parsed['env_before'].get('MOZ_PROFILE_GENERATE') == '1':
+                    # If MOZ_OBJDIR is specified in the mozconfig, we need to
+                    # make sure that the '/instrumented' directory gets appended
+                    # for the first build to avoid an objdir mismatch when
+                    # running 'mach package' on Windows.
+                    result['topobjdir'] = mozpath.join(result['topobjdir'], 'instrumented')
                 continue
 
             result['make_extra'].append(o)