Bug 1416052 - Check clobber state from Python; r=nalexander

The clobber logic is already written in Python. Now that we
always use mach in front of client.mk, we can check the clobber
state before we execute client.mk.

Since we always check the clobber state, we can remove the
CLOBBER files from various dependencies in client.mk. The
clobberer code should ensure everything is in a good state.

The refactor of the clobber Python code required some changes to
its testing. We drop some support for verifying output strings.
But testing this correctly would require a bit of effort. I don't
think it is worth it.

MozReview-Commit-ID: 69CoImCgtNm

client.mk

@@ -58,10 +58,6 @@ include $(OBJDIR)/.mozconfig-client-mk
 # of mozconfig output lines.
 MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
 
-ifdef AUTOCLOBBER
-export AUTOCLOBBER=1
-endif
-
 ifdef MOZ_PARALLEL_BUILD
   MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
   MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
@@ -101,7 +97,7 @@ endif
 
 # For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
 MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(OBJDIR)/CLOBBER
+$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG)
 	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 
 # Include that makefile so that it is created. This should not actually change
@@ -153,7 +149,6 @@ $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 CONFIG_STATUS_DEPS := \
   $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
   $(CONFIGURES) \
-  $(TOPSRCDIR)/CLOBBER \
   $(TOPSRCDIR)/nsprpub/configure \
   $(TOPSRCDIR)/config/milestone.txt \
   $(TOPSRCDIR)/browser/config/version.txt \
@@ -181,14 +176,9 @@ else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-$(OBJDIR)/CLOBBER: $(TOPSRCDIR)/CLOBBER
-	$(PYTHON) $(TOPSRCDIR)/config/pythonpath.py -I $(TOPSRCDIR)/testing/mozbase/mozfile \
-	    $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
-  $(OBJDIR)/CLOBBER \
   configure-files \
   save-mozconfig \
   $(OBJDIR)/.mozconfig.json \

python/mozbuild/mozbuild/controller/building.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import getpass
+import io
 import json
 import logging
 import os
@@ -32,6 +33,9 @@
 
 import mozpack.path as mozpath
 
+from .clobber import (
+    Clobberer,
+)
 from ..base import (
     BuildEnvironmentNotFoundException,
     MozbuildObject,
@@ -1329,6 +1333,10 @@ def _run_client_mk(self, target=None, line_handler=None, jobs=0,
         append_env['CONFIG_GUESS'] = self.resolve_config_guess()
 
         mozconfig = self.mozconfig
+
+        if self._check_clobber(mozconfig, os.environ):
+            return 1
+
         mozconfig_client_mk = os.path.join(self.topobjdir,
                                            '.mozconfig-client-mk')
         with FileAvoidWrite(mozconfig_client_mk) as fh:
@@ -1359,3 +1367,32 @@ def _run_client_mk(self, target=None, line_handler=None, jobs=0,
                               silent=not verbose,
                               keep_going=keep_going,
                               append_env=append_env)
+
+    def _check_clobber(self, mozconfig, env):
+        auto_clobber = any([
+            env.get('AUTOCLOBBER', False),
+            (mozconfig['env'] or {}).get('added', {}).get('AUTOCLOBBER', False),
+            'AUTOCLOBBER=1' in (mozconfig['make_extra'] or []),
+        ])
+
+        clobberer = Clobberer(self.topsrcdir, self.topobjdir)
+        clobber_output = io.BytesIO()
+        res = clobberer.maybe_do_clobber(os.getcwd(), auto_clobber,
+                                         clobber_output)
+        clobber_output.seek(0)
+        for line in clobber_output.readlines():
+            self.log(logging.WARNING, 'clobber',
+                     {'msg': line.rstrip()}, '{msg}')
+
+        clobber_required, clobber_performed, clobber_message = res
+        if not clobber_required or clobber_performed:
+            if clobber_performed and env.get('TINDERBOX_OUTPUT'):
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': 'TinderboxPrint: auto clobber'}, '{msg}')
+        else:
+            for line in clobber_message.splitlines():
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': line.rstrip()}, '{msg}')
+            return True
+
+        return False

python/mozbuild/mozbuild/controller/clobber.py

@@ -204,34 +204,3 @@ def _message(self, reason):
 
         return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
             no_reason='  ' + reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
-    if len(args) != 2:
-        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
-        return 1
-
-    topsrcdir, topobjdir = args
-
-    if not os.path.isabs(topsrcdir):
-        topsrcdir = os.path.abspath(topsrcdir)
-
-    if not os.path.isabs(topobjdir):
-        topobjdir = os.path.abspath(topobjdir)
-
-    auto = True if env.get('AUTOCLOBBER', False) else False
-    clobber = Clobberer(topsrcdir, topobjdir)
-    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
-    if not required or performed:
-        if performed and env.get('TINDERBOX_OUTPUT'):
-            print('TinderboxPrint: auto clobber', file=fh)
-        return 0
-
-    print(message, file=fh)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-

python/mozbuild/mozbuild/test/controller/test_clobber.py

@@ -13,8 +13,15 @@
 
 from mozunit import main
 
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
+from mozbuild.base import (
+    MozbuildObject,
+)
+from mozbuild.controller.building import (
+    BuildDriver,
+)
+from mozbuild.controller.clobber import (
+    Clobberer,
+)
 
 
 class TestClobberer(unittest.TestCase):
@@ -193,19 +200,19 @@ def test_mozconfig_opt_in(self):
         if env.get('AUTOCLOBBER', False):
             del env['AUTOCLOBBER']
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 1)
-        self.assertIn('Automatic clobbering is not enabled', s.getvalue())
+        mbo = MozbuildObject(topsrcdir, None, None, topobjdir)
+        build = mbo._spawn(BuildDriver)
+
+        status = build._check_clobber(build.mozconfig, env)
+
+        self.assertEqual(status, True)
         self.assertTrue(os.path.exists(dummy_file))
 
         # Check auto clobber opt-in works
         env['AUTOCLOBBER'] = '1'
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 0)
-        self.assertIn('Successfully completed auto clobber', s.getvalue())
+        status = build._check_clobber(build.mozconfig, env)
+        self.assertFalse(status)
         self.assertFalse(os.path.exists(dummy_file))