tools: update gyp to b3cef02

Includes two patches for AIX. Adds support for both 32-bit and 64-bit
files[0] and uses -RPf for cp instead of -af (which is unsupported)[1]

[0] https://codereview.chromium.org/1319663007
[1] https://codereview.chromium.org/1368133002

PR-URL: #3487
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: Shigeki Ohtsu <ohtsu@iij.ad.jp>

Author: Imran Iqbal

tools/gyp/PRESUBMIT.py

@@ -125,14 +125,13 @@ def CheckChangeOnCommit(input_api, output_api):
 
 
 TRYBOTS = [
-    'gyp-win32',
-    'gyp-win64',
-    'gyp-linux',
-    'gyp-mac',
+    'linux_try',
+    'mac_try',
+    'win_try',
 ]
 
 
 def GetPreferredTryMasters(_, change):
   return {
-      'tryserver.nacl': { t: set(['defaulttests']) for t in TRYBOTS },
+      'client.gyp': { t: set(['defaulttests']) for t in TRYBOTS },
   }

tools/gyp/gyp_main.py

@@ -4,15 +4,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
 import sys
 
-# TODO(mark): sys.path manipulation is some temporary testing stuff.
-try:
-  import gyp
-except ImportError, e:
-  import os.path
-  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), 'pylib'))
-  import gyp
+# Make sure we're using the version of pylib in this repo, not one installed
+# elsewhere on the system.
+sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), 'pylib'))
+import gyp
 
 if __name__ == '__main__':
   sys.exit(gyp.script_main())

tools/gyp/pylib/gyp/MSVSSettings.py

@@ -708,10 +708,7 @@ def _ValidateSettings(validators, settings, stderr):
 _MSBuildOnly(_compile, 'BuildingInIDE', _boolean)
 _MSBuildOnly(_compile, 'CompileAsManaged',
              _Enumeration([], new=['false',
-                                   'true',  # /clr
-                                   'Pure',  # /clr:pure
-                                   'Safe',  # /clr:safe
-                                   'OldSyntax']))  # /clr:oldSyntax
+                                   'true']))  # /clr
 _MSBuildOnly(_compile, 'CreateHotpatchableImage', _boolean)  # /hotpatch
 _MSBuildOnly(_compile, 'MultiProcessorCompilation', _boolean)  # /MP
 _MSBuildOnly(_compile, 'PreprocessOutputPath', _string)  # /Fi

tools/gyp/pylib/gyp/MSVSSettings_test.py

@@ -296,7 +296,7 @@ def testValidateMSBuildSettings_settings(self):
             'BuildingInIDE': 'true',
             'CallingConvention': 'Cdecl',
             'CompileAs': 'CompileAsC',
-            'CompileAsManaged': 'Pure',
+            'CompileAsManaged': 'true',
             'CreateHotpatchableImage': 'true',
             'DebugInformationFormat': 'ProgramDatabase',
             'DisableLanguageExtensions': 'true',

tools/gyp/pylib/gyp/common.py

@@ -131,13 +131,20 @@ def QualifiedTarget(build_file, target, toolset):
 
 
 @memoize
-def RelativePath(path, relative_to):
+def RelativePath(path, relative_to, follow_path_symlink=True):
   # Assuming both |path| and |relative_to| are relative to the current
   # directory, returns a relative path that identifies path relative to
   # relative_to.
+  # If |follow_symlink_path| is true (default) and |path| is a symlink, then
+  # this method returns a path to the real file represented by |path|. If it is
+  # false, this method returns a path to the symlink. If |path| is not a
+  # symlink, this option has no effect.
 
   # Convert to normalized (and therefore absolute paths).
-  path = os.path.realpath(path)
+  if follow_path_symlink:
+    path = os.path.realpath(path)
+  else:
+    path = os.path.abspath(path)
   relative_to = os.path.realpath(relative_to)
 
   # On Windows, we can't create a relative path to a different drive, so just

tools/gyp/pylib/gyp/generator/analyzer.py

@@ -338,7 +338,7 @@ def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files,
       sources = _ExtractSources(target_name, target_dicts[target_name],
                                 toplevel_dir)
       for source in sources:
-        if source in files:
+        if _ToGypPath(os.path.normpath(source)) in files:
           print 'target', target_name, 'matches', source
           target.match_status = MATCH_STATUS_MATCHES
           matching_targets.append(target)
@@ -498,7 +498,7 @@ def _WasGypIncludeFileModified(params, files):
   files."""
   if params['options'].includes:
     for include in params['options'].includes:
-      if _ToGypPath(include) in files:
+      if _ToGypPath(os.path.normpath(include)) in files:
         print 'Include file modified, assuming all changed', include
         return True
   return False

tools/gyp/pylib/gyp/generator/make.py

@@ -211,10 +211,10 @@ def CalculateGeneratorInputInfo(params):
 
 LINK_COMMANDS_AIX = """\
 quiet_cmd_alink = AR($(TOOLSET)) $@
-cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^)
+cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) -X32_64 crs $@ $(filter %.o,$^)
 
 quiet_cmd_alink_thin = AR($(TOOLSET)) $@
-cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^)
+cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) -X32_64 crs $@ $(filter %.o,$^)
 
 quiet_cmd_link = LINK($(TOOLSET)) $@
 cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS)
@@ -273,9 +273,9 @@ def CalculateGeneratorInputInfo(params):
 %(make_global_settings)s
 
 CC.target ?= %(CC.target)s
-CFLAGS.target ?= $(CFLAGS)
+CFLAGS.target ?= $(CPPFLAGS) $(CFLAGS)
 CXX.target ?= %(CXX.target)s
-CXXFLAGS.target ?= $(CXXFLAGS)
+CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS)
 LINK.target ?= %(LINK.target)s
 LDFLAGS.target ?= $(LDFLAGS)
 AR.target ?= $(AR)
@@ -286,9 +286,9 @@ def CalculateGeneratorInputInfo(params):
 # TODO(evan): move all cross-compilation logic to gyp-time so we don't need
 # to replicate this environment fallback in make as well.
 CC.host ?= %(CC.host)s
-CFLAGS.host ?=
+CFLAGS.host ?= $(CPPFLAGS_host) $(CFLAGS_host)
 CXX.host ?= %(CXX.host)s
-CXXFLAGS.host ?=
+CXXFLAGS.host ?= $(CPPFLAGS_host) $(CXXFLAGS_host)
 LINK.host ?= %(LINK.host)s
 LDFLAGS.host ?=
 AR.host ?= %(AR.host)s
@@ -365,7 +365,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_copy = COPY $@
 # send stderr to /dev/null to ignore messages when linking directories.
-cmd_copy = ln -f "$<" "$@" 2>/dev/null || (rm -rf "$@" && cp -af "$<" "$@")
+cmd_copy = ln -f "$<" "$@" 2>/dev/null || (rm -rf "$@" && cp %(copy_archive_args)s "$<" "$@")
 
 %(link_commands)s
 """
@@ -2010,6 +2010,7 @@ def CalculateMakefilePath(build_file, base_name):
     srcdir_prefix = '$(srcdir)/'
 
   flock_command= 'flock'
+  copy_archive_arguments = '-af'
   header_params = {
       'default_target': default_target,
       'builddir': builddir_name,
@@ -2019,6 +2020,7 @@ def CalculateMakefilePath(build_file, base_name):
       'link_commands': LINK_COMMANDS_LINUX,
       'extra_commands': '',
       'srcdir': srcdir,
+      'copy_archive_args': copy_archive_arguments,
     }
   if flavor == 'mac':
     flock_command = './gyp-mac-tool flock'
@@ -2043,7 +2045,9 @@ def CalculateMakefilePath(build_file, base_name):
         'flock': 'lockf',
     })
   elif flavor == 'aix':
+    copy_archive_arguments = '-pPRf'
     header_params.update({
+        'copy_archive_args': copy_archive_arguments,
         'link_commands': LINK_COMMANDS_AIX,
         'flock': './gyp-flock-tool flock',
         'flock_index': 2,

tools/gyp/pylib/gyp/generator/msvs.py

@@ -87,6 +87,8 @@ def _import_OrderedDict():
     'msvs_requires_importlibrary',
     'msvs_enable_winphone',
     'msvs_application_type_revision',
+    'msvs_target_platform_version',
+    'msvs_target_platform_minversion',
 ]
 
 
@@ -2644,6 +2646,17 @@ def _GetMSBuildGlobalProperties(spec, guid, gyp_file_name):
     else:
       properties[0].append(['ApplicationTypeRevision', '8.1'])
 
+    if spec.get('msvs_target_platform_version'):
+      target_platform_version = spec.get('msvs_target_platform_version')
+      properties[0].append(['WindowsTargetPlatformVersion',
+                            target_platform_version])
+      if spec.get('msvs_target_platform_minversion'):
+        target_platform_minversion = spec.get('msvs_target_platform_minversion')
+        properties[0].append(['WindowsTargetPlatformMinVersion',
+                              target_platform_minversion])
+      else:
+        properties[0].append(['WindowsTargetPlatformMinVersion',
+                              target_platform_version])
     if spec.get('msvs_enable_winphone'):
       properties[0].append(['ApplicationType', 'Windows Phone'])
     else:

tools/gyp/pylib/gyp/generator/ninja.py

@@ -921,6 +921,11 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources,
                   os.environ.get('CFLAGS', '').split() + cflags_c)
       cflags_cc = (os.environ.get('CPPFLAGS', '').split() +
                    os.environ.get('CXXFLAGS', '').split() + cflags_cc)
+    elif self.toolset == 'host':
+      cflags_c = (os.environ.get('CPPFLAGS_host', '').split() +
+                  os.environ.get('CFLAGS_host', '').split() + cflags_c)
+      cflags_cc = (os.environ.get('CPPFLAGS_host', '').split() +
+                   os.environ.get('CXXFLAGS_host', '').split() + cflags_cc)
 
     defines = config.get('defines', []) + extra_defines
     self.WriteVariableList(ninja_file, 'defines',
@@ -1672,7 +1677,7 @@ def CommandWithWrapper(cmd, wrappers, prog):
 
 def GetDefaultConcurrentLinks():
   """Returns a best-guess for a number of concurrent links."""
-  pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
+  pool_size = int(os.environ.get('GYP_LINK_CONCURRENCY', 0))
   if pool_size:
     return pool_size
 
@@ -1696,8 +1701,10 @@ class MEMORYSTATUSEX(ctypes.Structure):
     stat.dwLength = ctypes.sizeof(stat)
     ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
 
-    mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30)))  # total / 4GB
-    hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
+    # VS 2015 uses 20% more working set than VS 2013 and can consume all RAM
+    # on a 64 GB machine.
+    mem_limit = max(1, stat.ullTotalPhys / (5 * (2 ** 30)))  # total / 5GB
+    hard_cap = max(1, int(os.environ.get('GYP_LINK_CONCURRENCY_MAX', 2**32)))
     return min(mem_limit, hard_cap)
   elif sys.platform.startswith('linux'):
     if os.path.exists("/proc/meminfo"):
@@ -2275,7 +2282,11 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
     if flavor == 'mac':
       gyp.xcode_emulation.MergeGlobalXcodeSettingsToSpec(data[build_file], spec)
 
-    build_file = gyp.common.RelativePath(build_file, options.toplevel_dir)
+    # If build_file is a symlink, we must not follow it because there's a chance
+    # it could point to a path above toplevel_dir, and we cannot correctly deal
+    # with that case at the moment.
+    build_file = gyp.common.RelativePath(build_file, options.toplevel_dir,
+                                         False)
 
     qualified_target_for_hash = gyp.common.QualifiedTarget(build_file, name,
                                                            toolset)

tools/gyp/pylib/gyp/input.py

@@ -57,7 +57,7 @@ def IsPathSection(section):
   # If section ends in one of the '=+?!' characters, it's applied to a section
   # without the trailing characters.  '/' is notably absent from this list,
   # because there's no way for a regular expression to be treated as a path.
-  while section[-1:] in '=+?!':
+  while section and section[-1:] in '=+?!':
     section = section[:-1]
 
   if section in path_sections:
@@ -893,20 +893,24 @@ def ExpandVariables(input, phase, variables, build_file):
         else:
           # Fix up command with platform specific workarounds.
           contents = FixupPlatformCommand(contents)
-          p = subprocess.Popen(contents, shell=use_shell,
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE,
-                               stdin=subprocess.PIPE,
-                               cwd=build_file_dir)
+          try:
+            p = subprocess.Popen(contents, shell=use_shell,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 stdin=subprocess.PIPE,
+                                 cwd=build_file_dir)
+          except Exception, e:
+            raise GypError("%s while executing command '%s' in %s" %
+                           (e, contents, build_file))
 
           p_stdout, p_stderr = p.communicate('')
 
           if p.wait() != 0 or p_stderr:
             sys.stderr.write(p_stderr)
             # Simulate check_call behavior, since check_call only exists
             # in python 2.5 and later.
-            raise GypError("Call to '%s' returned exit status %d." %
-                           (contents, p.returncode))
+            raise GypError("Call to '%s' returned exit status %d while in %s." %
+                           (contents, p.returncode, build_file))
           replacement = p_stdout.rstrip()
 
         cached_command_results[cache_key] = replacement