Make landmines work on local builds too

Moves (some of) gyp environment setup out of gyp_chromium into separate
module, and shares that between gyp_chromium and landmines.py.

landmines.py is added as the first entry in DEPS hooks so that it can
clobber the entire build directory before running other hooks that
extract/generate into the build dir.

Reland with fix for ios, and for clean pull.

R=iannucci@chromium.org
BUG=400011

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=289099

Review URL: https://codereview.chromium.org/457003004

Cr-Commit-Position: refs/heads/master@{#289546}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@289546 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -29,6 +29,7 @@
 .cproject
 .gdb_history
 .gdbinit
+.landmines
 .metadata
 .project
 .pydevproject

DEPS

@@ -586,6 +586,17 @@ skip_child_includes = [
 
 
 hooks = [
+  {
+    # This clobbers when necessary (based on get_landmines.py). It must be the
+    # first hook so that other things that get/generate into the output
+    # directory will not subsequently be clobbered.
+    "name": "landmines",
+    "pattern": ".",
+    "action": [
+        "python",
+        "src/build/landmines.py",
+    ],
+  },
   {
     # This downloads binaries for Native Client's newlib toolchain.
     # Done in lieu of building the toolchain from scratch as it can take

build/get_landmines.py

@@ -8,7 +8,6 @@
 (or a list of 'landmines').
 """
 
-import optparse
 import sys
 
 import landmine_utils
@@ -21,10 +20,9 @@
 platform = landmine_utils.platform
 
 
-def print_landmines(target):
+def print_landmines():
   """
   ALL LANDMINES ARE EMITTED FROM HERE.
-  target can be one of {'Release', 'Debug', 'Debug_x64', 'Release_x64'}.
   """
   if (distributor() == 'goma' and platform() == 'win32' and
       builder() == 'ninja'):
@@ -60,16 +58,7 @@ def print_landmines(target):
 
 
 def main():
-  parser = optparse.OptionParser()
-  parser.add_option('-t', '--target',
-                    help=='Target for which the landmines have to be emitted')
-
-  options, args = parser.parse_args()
-
-  if args:
-    parser.error('Unknown arguments %s' % args)
-
-  print_landmines(options.target)
+  print_landmines()
   return 0
 
 

build/gyp_chromium

@@ -8,7 +8,7 @@
 # is invoked by Chromium beyond what can be done in the gclient hooks.
 
 import glob
-import gyp_helper
+import gyp_environment
 import os
 import re
 import shlex
@@ -197,10 +197,6 @@ if __name__ == '__main__':
     args.append('-Ganalyzer_output_path=' + args.pop(0))
 
   if int(os.environ.get('GYP_CHROMIUM_NO_ACTION', 0)):
-    # Check for landmines (reasons to clobber the build) in any case.
-    print 'Running build/landmines.py...'
-    subprocess.check_call(
-        [sys.executable, os.path.join(script_dir, 'landmines.py')])
     print 'Skipping gyp_chromium due to GYP_CHROMIUM_NO_ACTION env var.'
     sys.exit(0)
 
@@ -225,8 +221,6 @@ if __name__ == '__main__':
     p.communicate()
     sys.exit(p.returncode)
 
-  gyp_helper.apply_chromium_gyp_env()
-
   # This could give false positives since it doesn't actually do real option
   # parsing.  Oh well.
   gyp_file_specified = False
@@ -235,6 +229,8 @@ if __name__ == '__main__':
       gyp_file_specified = True
       break
 
+  gyp_environment.SetEnvironment()
+
   # If we didn't get a file, check an env var, and then fall back to
   # assuming 'all.gyp' from the same directory as the script.
   if not gyp_file_specified:
@@ -264,21 +260,6 @@ if __name__ == '__main__':
     print 'Error: make gyp generator not supported (check GYP_GENERATORS).'
     sys.exit(1)
 
-  # Default to ninja on linux and windows, but only if no generator has
-  # explicitly been set.
-  # Also default to ninja on mac, but only when not building chrome/ios.
-  # . -f / --format has precedence over the env var, no need to check for it
-  # . set the env var only if it hasn't been set yet
-  # . chromium.gyp_env has been applied to os.environ at this point already
-  if sys.platform.startswith(('linux', 'win', 'freebsd')) and \
-      not os.environ.get('GYP_GENERATORS'):
-    os.environ['GYP_GENERATORS'] = 'ninja'
-  elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
-      not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
-    os.environ['GYP_GENERATORS'] = 'ninja'
-
-  vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
-
   # If CHROMIUM_GYP_SYNTAX_CHECK is set to 1, it will invoke gyp with --check
   # to enfore syntax checking.
   syntax_check = os.environ.get('CHROMIUM_GYP_SYNTAX_CHECK')
@@ -322,13 +303,7 @@ if __name__ == '__main__':
   gyp_rc = gyp.main(args)
 
   if not use_analyzer:
-    # Check for landmines (reasons to clobber the build). This must be run here,
-    # rather than a separate runhooks step so that any environment modifications
-    # from above are picked up.
-    print 'Running build/landmines.py...'
-    subprocess.check_call(
-      [sys.executable, os.path.join(script_dir, 'landmines.py')])
-
+    vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
     if vs2013_runtime_dll_dirs:
       x64_runtime, x86_runtime = vs2013_runtime_dll_dirs
       vs_toolchain.CopyVsRuntimeDlls(

build/gyp_environment.py

@@ -0,0 +1,33 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Sets up various automatic gyp environment variables. These are used by
+gyp_chromium and landmines.py which run at different stages of runhooks. To
+make sure settings are consistent between them, all setup should happen here.
+"""
+
+import gyp_helper
+import os
+import sys
+import vs_toolchain
+
+def SetEnvironment():
+  """Sets defaults for GYP_* variables."""
+  gyp_helper.apply_chromium_gyp_env()
+
+  # Default to ninja on linux and windows, but only if no generator has
+  # explicitly been set.
+  # Also default to ninja on mac, but only when not building chrome/ios.
+  # . -f / --format has precedence over the env var, no need to check for it
+  # . set the env var only if it hasn't been set yet
+  # . chromium.gyp_env has been applied to os.environ at this point already
+  if sys.platform.startswith(('linux', 'win', 'freebsd')) and \
+      not os.environ.get('GYP_GENERATORS'):
+    os.environ['GYP_GENERATORS'] = 'ninja'
+  elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
+      not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
+    os.environ['GYP_GENERATORS'] = 'ninja'
+
+  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()

build/landmines.py

@@ -4,10 +4,8 @@
 # found in the LICENSE file.
 
 """
-This script runs every build as a hook. If it detects that the build should
-be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
-various build scripts will then check for the presence of this file and clobber
-accordingly. The script will also emit the reasons for the clobber to stdout.
+This script runs every build as the first hook (See DEPS). If it detects that
+the build should be clobbered, it will remove the build directory.
 
 A landmine is tripped when a builder checks out a different revision, and the
 diff between the new landmines and the old ones is non-null. At this point, the
@@ -16,9 +14,11 @@
 
 import difflib
 import errno
+import gyp_environment
 import logging
 import optparse
 import os
+import shutil
 import sys
 import subprocess
 import time
@@ -29,56 +29,53 @@
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
-def get_target_build_dir(build_tool, target, is_iphone=False):
+def get_build_dir(build_tool, is_iphone=False):
   """
   Returns output directory absolute path dependent on build and targets.
   Examples:
-    r'c:\b\build\slave\win\build\src\out\Release'
-    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
-    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+    r'c:\b\build\slave\win\build\src\out'
+    '/mnt/data/b/build/slave/linux/build/src/out'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild'
 
   Keep this function in sync with tools/build/scripts/slave/compile.py
   """
   ret = None
   if build_tool == 'xcode':
-    ret = os.path.join(SRC_DIR, 'xcodebuild',
-        target + ('-iphoneos' if is_iphone else ''))
+    ret = os.path.join(SRC_DIR, 'xcodebuild')
   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
-    ret = os.path.join(SRC_DIR, 'out', target)
+    ret = os.path.join(SRC_DIR, 'out')
   elif build_tool in ['msvs', 'vs', 'ib']:
-    ret = os.path.join(SRC_DIR, 'build', target)
+    ret = os.path.join(SRC_DIR, 'build')
   else:
     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
   return os.path.abspath(ret)
 
 
-def set_up_landmines(target, new_landmines):
+def clobber_if_necessary(new_landmines):
   """Does the work of setting, planting, and triggering landmines."""
-  out_dir = get_target_build_dir(landmine_utils.builder(), target,
-                                 landmine_utils.platform() == 'ios')
-
-  landmines_path = os.path.join(out_dir, '.landmines')
+  out_dir = get_build_dir(landmine_utils.builder())
+  landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
   try:
     os.makedirs(out_dir)
   except OSError as e:
     if e.errno == errno.EEXIST:
       pass
 
   if os.path.exists(landmines_path):
-    triggered = os.path.join(out_dir, '.landmines_triggered')
     with open(landmines_path, 'r') as f:
       old_landmines = f.readlines()
     if old_landmines != new_landmines:
       old_date = time.ctime(os.stat(landmines_path).st_ctime)
       diff = difflib.unified_diff(old_landmines, new_landmines,
           fromfile='old_landmines', tofile='new_landmines',
           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
+      sys.stdout.write('Clobbering due to:\n')
+      sys.stdout.writelines(diff)
+
+      # Clobber.
+      shutil.rmtree(out_dir)
 
-      with open(triggered, 'w') as f:
-        f.writelines(diff)
-    elif os.path.exists(triggered):
-      # Remove false triggered landmines.
-      os.remove(triggered)
+  # Save current set of landmines for next time.
   with open(landmines_path, 'w') as f:
     f.writelines(new_landmines)
 
@@ -119,14 +116,14 @@ def main():
   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
     return 0
 
-  for target in ('Debug', 'Release', 'Debug_x64', 'Release_x64'):
-    landmines = []
-    for s in landmine_scripts:
-      proc = subprocess.Popen([sys.executable, s, '-t', target],
-                              stdout=subprocess.PIPE)
-      output, _ = proc.communicate()
-      landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
-    set_up_landmines(target, landmines)
+  gyp_environment.SetEnvironment()
+
+  landmines = []
+  for s in landmine_scripts:
+    proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
+    output, _ = proc.communicate()
+    landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
+  clobber_if_necessary(landmines)
 
   return 0
 

build/vs_toolchain.py

@@ -30,6 +30,8 @@ def SetEnvironmentAndGetRuntimeDllDirs():
   depot_tools_win_toolchain = \
       bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')))
   if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain:
+    if not os.path.exists(json_data_file):
+      Update()
     with open(json_data_file, 'r') as tempf:
       toolchain_data = json.load(tempf)