diff --git a/tools/gyp/PRESUBMIT.py b/tools/gyp/PRESUBMIT.py index 9c474eb2b106..acdb34b2bb2e 100644 --- a/tools/gyp/PRESUBMIT.py +++ b/tools/gyp/PRESUBMIT.py @@ -16,8 +16,6 @@ 'test/lib/TestCmd.py', 'test/lib/TestCommon.py', 'test/lib/TestGyp.py', - # Needs style fix. - 'pylib/gyp/generator/xcode.py', ] @@ -25,6 +23,10 @@ # TODO: fix me. # Many tests include modules they don't use. 'W0611', + # Possible unbalanced tuple unpacking with sequence. + 'W0632', + # Attempting to unpack a non-sequence. + 'W0633', # Include order doesn't properly include local files? 'F0401', # Some use of built-in names. @@ -40,6 +42,10 @@ 'W0613', # String has no effect (docstring in wrong place). 'W0105', + # map/filter on lambda could be replaced by comprehension. + 'W0110', + # Use of eval. + 'W0123', # Comma not followed by space. 'C0324', # Access to a protected member. @@ -56,6 +62,8 @@ 'E1101', # Dangerous default {}. 'W0102', + # Cyclic import. + 'R0401', # Others, too many to sort. 'W0201', 'W0232', 'E1103', 'W0621', 'W0108', 'W0223', 'W0231', 'R0201', 'E0101', 'C0321', @@ -116,5 +124,15 @@ def CheckChangeOnCommit(input_api, output_api): return report -def GetPreferredTrySlaves(): - return ['gyp-win32', 'gyp-win64', 'gyp-linux', 'gyp-mac', 'gyp-android'] +TRYBOTS = [ + 'gyp-win32', + 'gyp-win64', + 'gyp-linux', + 'gyp-mac', +] + + +def GetPreferredTryMasters(_, change): + return { + 'tryserver.nacl': { t: set(['defaulttests']) for t in TRYBOTS }, + } diff --git a/tools/gyp/buildbot/buildbot_run.py b/tools/gyp/buildbot/buildbot_run.py index 979073c773be..9a2b71f1b355 100755 --- a/tools/gyp/buildbot/buildbot_run.py +++ b/tools/gyp/buildbot/buildbot_run.py @@ -3,26 +3,17 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Argument-less script to select what to run on the buildbots.""" - import os import shutil import subprocess import sys -if sys.platform in ['win32', 'cygwin']: - EXE_SUFFIX = '.exe' -else: - EXE_SUFFIX = '' - - BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__)) TRUNK_DIR = os.path.dirname(BUILDBOT_DIR) ROOT_DIR = os.path.dirname(TRUNK_DIR) -ANDROID_DIR = os.path.join(ROOT_DIR, 'android') CMAKE_DIR = os.path.join(ROOT_DIR, 'cmake') CMAKE_BIN_DIR = os.path.join(CMAKE_DIR, 'bin') OUT_DIR = os.path.join(TRUNK_DIR, 'out') @@ -30,7 +21,8 @@ def CallSubProcess(*args, **kwargs): """Wrapper around subprocess.call which treats errors as build exceptions.""" - retcode = subprocess.call(*args, **kwargs) + with open(os.devnull) as devnull_fd: + retcode = subprocess.call(stdin=devnull_fd, *args, **kwargs) if retcode != 0: print '@@@STEP_EXCEPTION@@@' sys.exit(1) @@ -49,10 +41,6 @@ def PrepareCmake(): print '@@@BUILD_STEP Initialize CMake checkout@@@' os.mkdir(CMAKE_DIR) - CallSubProcess(['git', 'config', '--global', 'user.name', 'trybot']) - CallSubProcess(['git', 'config', '--global', - 'user.email', 'chrome-bot@google.com']) - CallSubProcess(['git', 'config', '--global', 'color.ui', 'false']) print '@@@BUILD_STEP Sync CMake@@@' CallSubProcess( @@ -73,41 +61,7 @@ def PrepareCmake(): CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR) -def PrepareAndroidTree(): - """Prepare an Android tree to run 'android' format tests.""" - if os.environ['BUILDBOT_CLOBBER'] == '1': - print '@@@BUILD_STEP Clobber Android checkout@@@' - shutil.rmtree(ANDROID_DIR) - - # The release of Android we use is static, so there's no need to do anything - # if the directory already exists. - if os.path.isdir(ANDROID_DIR): - return - - print '@@@BUILD_STEP Initialize Android checkout@@@' - os.mkdir(ANDROID_DIR) - CallSubProcess(['git', 'config', '--global', 'user.name', 'trybot']) - CallSubProcess(['git', 'config', '--global', - 'user.email', 'chrome-bot@google.com']) - CallSubProcess(['git', 'config', '--global', 'color.ui', 'false']) - CallSubProcess( - ['repo', 'init', - '-u', 'https://android.googlesource.com/platform/manifest', - '-b', 'android-4.2.1_r1', - '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'], - cwd=ANDROID_DIR) - - print '@@@BUILD_STEP Sync Android@@@' - CallSubProcess(['repo', 'sync', '-j4'], cwd=ANDROID_DIR) - - print '@@@BUILD_STEP Build Android@@@' - CallSubProcess( - ['/bin/bash', - '-c', 'source build/envsetup.sh && lunch full-eng && make -j4'], - cwd=ANDROID_DIR) - - -def GypTestFormat(title, format=None, msvs_version=None): +def GypTestFormat(title, format=None, msvs_version=None, tests=[]): """Run the gyp tests for a given format, emitting annotator tags. See annotator docs at: @@ -126,22 +80,13 @@ def GypTestFormat(title, format=None, msvs_version=None): if msvs_version: env['GYP_MSVS_VERSION'] = msvs_version command = ' '.join( - [sys.executable, 'trunk/gyptest.py', + [sys.executable, 'gyp/gyptest.py', '--all', '--passed', '--format', format, '--path', CMAKE_BIN_DIR, - '--chdir', 'trunk']) - if format == 'android': - # gyptest needs the environment setup from envsetup/lunch in order to build - # using the 'android' backend, so this is done in a single shell. - retcode = subprocess.call( - ['/bin/bash', - '-c', 'source build/envsetup.sh && lunch full-eng && cd %s && %s' - % (ROOT_DIR, command)], - cwd=ANDROID_DIR, env=env) - else: - retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True) + '--chdir', 'gyp'] + tests) + retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True) if retcode: # Emit failure tag, and keep going. print '@@@STEP_FAILURE@@@' @@ -157,11 +102,7 @@ def GypBuild(): print 'Done.' retcode = 0 - # The Android gyp bot runs on linux so this must be tested first. - if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android': - PrepareAndroidTree() - retcode += GypTestFormat('android') - elif sys.platform.startswith('linux'): + if sys.platform.startswith('linux'): retcode += GypTestFormat('ninja') retcode += GypTestFormat('make') PrepareCmake() @@ -173,8 +114,13 @@ def GypBuild(): elif sys.platform == 'win32': retcode += GypTestFormat('ninja') if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-win64': - retcode += GypTestFormat('msvs-2010', format='msvs', msvs_version='2010') - retcode += GypTestFormat('msvs-2012', format='msvs', msvs_version='2012') + retcode += GypTestFormat('msvs-ninja-2013', format='msvs-ninja', + msvs_version='2013', + tests=[ + r'test\generator-output\gyptest-actions.py', + r'test\generator-output\gyptest-relocate.py', + r'test\generator-output\gyptest-rules.py']) + retcode += GypTestFormat('msvs-2013', format='msvs', msvs_version='2013') else: raise Exception('Unknown platform') if retcode: diff --git a/tools/gyp/buildbot/commit_queue/OWNERS b/tools/gyp/buildbot/commit_queue/OWNERS new file mode 100644 index 000000000000..b269c198b43e --- /dev/null +++ b/tools/gyp/buildbot/commit_queue/OWNERS @@ -0,0 +1,6 @@ +set noparent +bradnelson@chromium.org +bradnelson@google.com +iannucci@chromium.org +scottmg@chromium.org +thakis@chromium.org diff --git a/tools/gyp/buildbot/commit_queue/README b/tools/gyp/buildbot/commit_queue/README new file mode 100644 index 000000000000..942849788327 --- /dev/null +++ b/tools/gyp/buildbot/commit_queue/README @@ -0,0 +1,3 @@ +cq_config.json describes the trybots that must pass in order +to land a change through the commit queue. +Comments are here as the file is strictly JSON. diff --git a/tools/gyp/buildbot/commit_queue/cq_config.json b/tools/gyp/buildbot/commit_queue/cq_config.json new file mode 100644 index 000000000000..656c21e54fb1 --- /dev/null +++ b/tools/gyp/buildbot/commit_queue/cq_config.json @@ -0,0 +1,15 @@ +{ + "trybots": { + "launched": { + "tryserver.nacl": { + "gyp-presubmit": ["defaulttests"], + "gyp-linux": ["defaulttests"], + "gyp-mac": ["defaulttests"], + "gyp-win32": ["defaulttests"], + "gyp-win64": ["defaulttests"] + } + }, + "triggered": { + } + } +} diff --git a/tools/gyp/codereview.settings b/tools/gyp/codereview.settings index a04a2440df39..faf37f1145f1 100644 --- a/tools/gyp/codereview.settings +++ b/tools/gyp/codereview.settings @@ -1,10 +1,10 @@ # This file is used by gcl to get repository specific information. CODE_REVIEW_SERVER: codereview.chromium.org CC_LIST: gyp-developer@googlegroups.com -VIEW_VC: http://code.google.com/p/gyp/source/detail?r= -TRY_ON_UPLOAD: True +VIEW_VC: https://chromium.googlesource.com/external/gyp/+/ +TRY_ON_UPLOAD: False TRYSERVER_PROJECT: gyp -TRYSERVER_PATCHLEVEL: 0 -TRYSERVER_ROOT: trunk +TRYSERVER_PATCHLEVEL: 1 +TRYSERVER_ROOT: gyp TRYSERVER_SVN_URL: svn://svn.chromium.org/chrome-try/try-nacl - +PROJECT: gyp diff --git a/tools/gyp/gyp b/tools/gyp/gyp index b53a6dde8f2d..1b8b9bdfb05f 100755 --- a/tools/gyp/gyp +++ b/tools/gyp/gyp @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/tools/gyp/gyp_dummy.c b/tools/gyp/gyp_dummy.c deleted file mode 100644 index fb55bbc78da2..000000000000 --- a/tools/gyp/gyp_dummy.c +++ /dev/null @@ -1,7 +0,0 @@ -/* Copyright (c) 2009 Google Inc. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. */ - -int main() { - return 0; -} diff --git a/tools/gyp/gyptest.py b/tools/gyp/gyptest.py index 8f3ee0ffb04f..8e4fc47d5c80 100755 --- a/tools/gyp/gyptest.py +++ b/tools/gyp/gyptest.py @@ -13,7 +13,7 @@ import subprocess import sys -class CommandRunner: +class CommandRunner(object): """ Executor class for commands, including "commands" implemented by Python functions. @@ -117,7 +117,7 @@ def run(self, command, display=None, stdout=None, stderr=None): return self.execute(command, stdout, stderr) -class Unbuffered: +class Unbuffered(object): def __init__(self, fp): self.fp = fp def write(self, arg): @@ -224,7 +224,7 @@ def main(argv=None): 'win32': ['msvs', 'ninja'], 'linux2': ['make', 'ninja'], 'linux3': ['make', 'ninja'], - 'darwin': ['make', 'ninja', 'xcode'], + 'darwin': ['make', 'ninja', 'xcode', 'xcode-ninja'], }[sys.platform] for format in format_list: diff --git a/tools/gyp/pylib/gyp/MSVSNew.py b/tools/gyp/pylib/gyp/MSVSNew.py index 845dcb0639d7..593f0e5b0b2e 100644 --- a/tools/gyp/pylib/gyp/MSVSNew.py +++ b/tools/gyp/pylib/gyp/MSVSNew.py @@ -172,7 +172,7 @@ def set_msbuild_toolset(self, msbuild_toolset): #------------------------------------------------------------------------------ -class MSVSSolution: +class MSVSSolution(object): """Visual Studio solution.""" def __init__(self, path, version, entries=None, variants=None, diff --git a/tools/gyp/pylib/gyp/MSVSSettings.py b/tools/gyp/pylib/gyp/MSVSSettings.py index 0c9532d8e677..dde0e07092b1 100644 --- a/tools/gyp/pylib/gyp/MSVSSettings.py +++ b/tools/gyp/pylib/gyp/MSVSSettings.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Code to validate and convert settings of the Microsoft build tools. +r"""Code to validate and convert settings of the Microsoft build tools. This file contains code to validate and convert settings of the Microsoft build tools. The function ConvertToMSBuildSettings(), ValidateMSVSSettings(), @@ -314,7 +314,14 @@ def _MSBuildOnly(tool, name, setting_type): name: the name of the setting. setting_type: the type of this setting. """ + + def _Translate(value, msbuild_settings): + # Let msbuild-only properties get translated as-is from msvs_settings. + tool_settings = msbuild_settings.setdefault(tool.msbuild_name, {}) + tool_settings[name] = value + _msbuild_validators[tool.msbuild_name][name] = setting_type.ValidateMSBuild + _msvs_to_msbuild_converters[tool.msvs_name][name] = _Translate def _ConvertedToAdditionalOption(tool, msvs_name, flag): @@ -367,6 +374,35 @@ def _Translate(value, msbuild_settings): r'(\$\((?:%s)\))(?:[\\/]+)' % "|".join(fix_vc_macro_slashes_regex_list) ) +# Regular expression to detect keys that were generated by exclusion lists +_EXCLUDED_SUFFIX_RE = re.compile('^(.*)_excluded$') + + +def _ValidateExclusionSetting(setting, settings, error_msg, stderr=sys.stderr): + """Verify that 'setting' is valid if it is generated from an exclusion list. + + If the setting appears to be generated from an exclusion list, the root name + is checked. + + Args: + setting: A string that is the setting name to validate + settings: A dictionary where the keys are valid settings + error_msg: The message to emit in the event of error + stderr: The stream receiving the error messages. + """ + # This may be unrecognized because it's an exclusion list. If the + # setting name has the _excluded suffix, then check the root name. + unrecognized = True + m = re.match(_EXCLUDED_SUFFIX_RE, setting) + if m: + root_setting = m.group(1) + unrecognized = root_setting not in settings + + if unrecognized: + # We don't know this setting. Give a warning. + print >> stderr, error_msg + + def FixVCMacroSlashes(s): """Replace macros which have excessive following slashes. @@ -388,11 +424,11 @@ def ConvertVCMacrosToMSBuild(s): if '$' in s: replace_map = { '$(ConfigurationName)': '$(Configuration)', - '$(InputDir)': '%(RootDir)%(Directory)', + '$(InputDir)': '%(RelativeDir)', '$(InputExt)': '%(Extension)', '$(InputFileName)': '%(Filename)%(Extension)', '$(InputName)': '%(Filename)', - '$(InputPath)': '%(FullPath)', + '$(InputPath)': '%(Identity)', '$(ParentName)': '$(ProjectFileName)', '$(PlatformName)': '$(Platform)', '$(SafeInputName)': '%(Filename)', @@ -429,10 +465,12 @@ def ConvertToMSBuildSettings(msvs_settings, stderr=sys.stderr): print >> stderr, ('Warning: while converting %s/%s to MSBuild, ' '%s' % (msvs_tool_name, msvs_setting, e)) else: - # We don't know this setting. Give a warning. - print >> stderr, ('Warning: unrecognized setting %s/%s ' - 'while converting to MSBuild.' % - (msvs_tool_name, msvs_setting)) + _ValidateExclusionSetting(msvs_setting, + msvs_tool, + ('Warning: unrecognized setting %s/%s ' + 'while converting to MSBuild.' % + (msvs_tool_name, msvs_setting)), + stderr) else: print >> stderr, ('Warning: unrecognized tool %s while converting to ' 'MSBuild.' % msvs_tool_name) @@ -483,8 +521,12 @@ def _ValidateSettings(validators, settings, stderr): print >> stderr, ('Warning: for %s/%s, %s' % (tool_name, setting, e)) else: - print >> stderr, ('Warning: unrecognized setting %s/%s' % - (tool_name, setting)) + _ValidateExclusionSetting(setting, + tool_validators, + ('Warning: unrecognized setting %s/%s' % + (tool_name, setting)), + stderr) + else: print >> stderr, ('Warning: unrecognized tool %s' % tool_name) @@ -496,6 +538,7 @@ def _ValidateSettings(validators, settings, stderr): _rc = _Tool('VCResourceCompilerTool', 'ResourceCompile') _lib = _Tool('VCLibrarianTool', 'Lib') _manifest = _Tool('VCManifestTool', 'Manifest') +_masm = _Tool('MASM', 'MASM') _AddTool(_compile) @@ -504,6 +547,7 @@ def _ValidateSettings(validators, settings, stderr): _AddTool(_rc) _AddTool(_lib) _AddTool(_manifest) +_AddTool(_masm) # Add sections only found in the MSBuild settings. _msbuild_validators[''] = {} _msbuild_validators['ProjectReference'] = {} @@ -567,7 +611,8 @@ def _ValidateSettings(validators, settings, stderr): _Same(_compile, 'CallingConvention', _Enumeration(['Cdecl', # /Gd 'FastCall', # /Gr - 'StdCall'])) # /Gz + 'StdCall', # /Gz + 'VectorCall'])) # /Gv _Same(_compile, 'CompileAs', _Enumeration(['Default', 'CompileAsC', # /TC @@ -581,7 +626,12 @@ def _ValidateSettings(validators, settings, stderr): _Same(_compile, 'EnableEnhancedInstructionSet', _Enumeration(['NotSet', 'StreamingSIMDExtensions', # /arch:SSE - 'StreamingSIMDExtensions2'])) # /arch:SSE2 + 'StreamingSIMDExtensions2', # /arch:SSE2 + 'AdvancedVectorExtensions', # /arch:AVX (vs2012+) + 'NoExtensions', # /arch:IA32 (vs2012+) + # This one only exists in the new msbuild format. + 'AdvancedVectorExtensions2', # /arch:AVX2 (vs2013r2+) + ])) _Same(_compile, 'ErrorReporting', _Enumeration(['None', # /errorReport:none 'Prompt', # /errorReport:prompt @@ -836,13 +886,6 @@ def _ValidateSettings(validators, settings, stderr): # MSVS options not found in MSBuild. _MSVSOnly(_link, 'OptimizeForWindows98', _newly_boolean) _MSVSOnly(_link, 'UseUnicodeResponseFiles', _boolean) -# These settings generate correctly in the MSVS output files when using -# e.g. DelayLoadDLLs! or AdditionalDependencies! to exclude files from -# configuration entries, but result in spurious artifacts which can be -# safely ignored here. See crbug.com/246570 -_MSVSOnly(_link, 'AdditionalLibraryDirectories_excluded', _folder_list) -_MSVSOnly(_link, 'DelayLoadDLLs_excluded', _file_list) -_MSVSOnly(_link, 'AdditionalDependencies_excluded', _file_list) # MSBuild options not found in MSVS. _MSBuildOnly(_link, 'BuildingInIDE', _boolean) @@ -991,9 +1034,6 @@ def _ValidateSettings(validators, settings, stderr): # ProjectReference. We may want to validate that they are consistent. _Moved(_lib, 'LinkLibraryDependencies', 'ProjectReference', _boolean) -# TODO(jeanluc) I don't think these are genuine settings but byproducts of Gyp. -_MSVSOnly(_lib, 'AdditionalLibraryDirectories_excluded', _folder_list) - _MSBuildOnly(_lib, 'DisplayLibrary', _string) # /LIST Visible='false' _MSBuildOnly(_lib, 'ErrorReporting', _Enumeration([], new=['PromptImmediately', # /ERRORREPORT:PROMPT @@ -1049,3 +1089,11 @@ def _ValidateSettings(validators, settings, stderr): _MSBuildOnly(_manifest, 'OutputResourceManifests', _string) # /outputresource _MSBuildOnly(_manifest, 'SuppressDependencyElement', _boolean) # /nodependency _MSBuildOnly(_manifest, 'TrackerLogDirectory', _folder_name) + + +# Directives for MASM. +# See "$(VCTargetsPath)\BuildCustomizations\masm.xml" for the schema of the +# MSBuild MASM settings. + +# Options that have the same name in MSVS and MSBuild. +_Same(_masm, 'UseSafeExceptionHandlers', _boolean) # /safeseh diff --git a/tools/gyp/pylib/gyp/MSVSSettings_test.py b/tools/gyp/pylib/gyp/MSVSSettings_test.py index 4e06da3bbe3f..d24dcac4d5e1 100755 --- a/tools/gyp/pylib/gyp/MSVSSettings_test.py +++ b/tools/gyp/pylib/gyp/MSVSSettings_test.py @@ -109,6 +109,7 @@ def testValidateMSVSSettings_settings(self): 'ZZXYZ': 'bogus'}, 'VCLinkerTool': { 'AdditionalDependencies': 'file1;file2', + 'AdditionalDependencies_excluded': 'file3', 'AdditionalLibraryDirectories': 'folder1;folder2', 'AdditionalManifestDependencies': 'file1;file2', 'AdditionalOptions': 'a string1', @@ -266,7 +267,7 @@ def testValidateMSVSSettings_settings(self): 'Warning: for VCCLCompilerTool/BrowseInformation, ' "invalid literal for int() with base 10: 'fdkslj'", 'Warning: for VCCLCompilerTool/CallingConvention, ' - 'index value (-1) not in expected range [0, 3)', + 'index value (-1) not in expected range [0, 4)', 'Warning: for VCCLCompilerTool/DebugInformationFormat, ' 'converted value for 2 not specified.', 'Warning: unrecognized setting VCCLCompilerTool/Enableprefast', diff --git a/tools/gyp/pylib/gyp/MSVSUtil.py b/tools/gyp/pylib/gyp/MSVSUtil.py index 62e8d260d444..0b32e9118078 100644 --- a/tools/gyp/pylib/gyp/MSVSUtil.py +++ b/tools/gyp/pylib/gyp/MSVSUtil.py @@ -8,10 +8,12 @@ import os -_TARGET_TYPE_EXT = { - 'executable': '.exe', - 'loadable_module': '.dll', - 'shared_library': '.dll', +# A dictionary mapping supported target types to extensions. +TARGET_TYPE_EXT = { + 'executable': 'exe', + 'loadable_module': 'dll', + 'shared_library': 'dll', + 'static_library': 'lib', } @@ -109,15 +111,16 @@ def ShardTargets(target_list, target_dicts): new_target_dicts[t] = target_dicts[t] # Shard dependencies. for t in new_target_dicts: - dependencies = copy.copy(new_target_dicts[t].get('dependencies', [])) - new_dependencies = [] - for d in dependencies: - if d in targets_to_shard: - for i in range(targets_to_shard[d]): - new_dependencies.append(_ShardName(d, i)) - else: - new_dependencies.append(d) - new_target_dicts[t]['dependencies'] = new_dependencies + for deptype in ('dependencies', 'dependencies_original'): + dependencies = copy.copy(new_target_dicts[t].get(deptype, [])) + new_dependencies = [] + for d in dependencies: + if d in targets_to_shard: + for i in range(targets_to_shard[d]): + new_dependencies.append(_ShardName(d, i)) + else: + new_dependencies.append(d) + new_target_dicts[t][deptype] = new_dependencies return (new_target_list, new_target_dicts) @@ -156,7 +159,7 @@ def _GetPdbPath(target_dict, config_name, vars): pdb_base = target_dict.get('product_name', target_dict['target_name']) - pdb_base = '%s%s.pdb' % (pdb_base, _TARGET_TYPE_EXT[target_dict['type']]) + pdb_base = '%s.%s.pdb' % (pdb_base, TARGET_TYPE_EXT[target_dict['type']]) pdb_path = vars['PRODUCT_DIR'] + '/' + pdb_base return pdb_path @@ -264,4 +267,4 @@ def InsertLargePdbShims(target_list, target_dicts, vars): # Update the original target to depend on the shim target. target_dict.setdefault('dependencies', []).append(full_shim_target_name) - return (target_list, target_dicts) \ No newline at end of file + return (target_list, target_dicts) diff --git a/tools/gyp/pylib/gyp/MSVSVersion.py b/tools/gyp/pylib/gyp/MSVSVersion.py index 03b6d8ad42cc..d9bfa684fa30 100644 --- a/tools/gyp/pylib/gyp/MSVSVersion.py +++ b/tools/gyp/pylib/gyp/MSVSVersion.py @@ -84,10 +84,11 @@ def SetupScript(self, target_arch): # vcvars32, which it can only find if VS??COMNTOOLS is set, which it # isn't always. if target_arch == 'x86': - if self.short_name == '2013' and ( + if self.short_name >= '2013' and self.short_name[-1] != 'e' and ( os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'): - # VS2013 non-Express has a x64-x86 cross that we want to prefer. + # VS2013 and later, non-Express have a x64-x86 cross that we want + # to prefer. return [os.path.normpath( os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86'] # Otherwise, the standard x86 compiler. @@ -138,7 +139,7 @@ def _RegistryQueryBase(sysdir, key, value): def _RegistryQuery(key, value=None): - """Use reg.exe to read a particular key through _RegistryQueryBase. + r"""Use reg.exe to read a particular key through _RegistryQueryBase. First tries to launch from %WinDir%\Sysnative to avoid WoW64 redirection. If that fails, it falls back to System32. Sysnative is available on Vista and @@ -165,8 +166,33 @@ def _RegistryQuery(key, value=None): return text +def _RegistryGetValueUsingWinReg(key, value): + """Use the _winreg module to obtain the value of a registry key. + + Args: + key: The registry key. + value: The particular registry value to read. + Return: + contents of the registry key's value, or None on failure. Throws + ImportError if _winreg is unavailable. + """ + import _winreg + try: + root, subkey = key.split('\\', 1) + assert root == 'HKLM' # Only need HKLM for now. + with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, subkey) as hkey: + return _winreg.QueryValueEx(hkey, value)[0] + except WindowsError: + return None + + def _RegistryGetValue(key, value): - """Use reg.exe to obtain the value of a registry key. + """Use _winreg or reg.exe to obtain the value of a registry key. + + Using _winreg is preferable because it solves an issue on some corporate + environments where access to reg.exe is locked down. However, we still need + to fallback to reg.exe for the case where the _winreg module is not available + (for example in cygwin python). Args: key: The registry key. @@ -174,6 +200,12 @@ def _RegistryGetValue(key, value): Return: contents of the registry key's value, or None on failure. """ + try: + return _RegistryGetValueUsingWinReg(key, value) + except ImportError: + pass + + # Fallback to reg.exe if we fail to import _winreg. text = _RegistryQuery(key, value) if not text: return None @@ -184,19 +216,6 @@ def _RegistryGetValue(key, value): return match.group(1) -def _RegistryKeyExists(key): - """Use reg.exe to see if a key exists. - - Args: - key: The registry key to check. - Return: - True if the key exists - """ - if not _RegistryQuery(key): - return False - return True - - def _CreateVersion(name, path, sdk_based=False): """Sets up MSVS project generation. @@ -207,6 +226,15 @@ def _CreateVersion(name, path, sdk_based=False): if path: path = os.path.normpath(path) versions = { + '2015': VisualStudioVersion('2015', + 'Visual Studio 2015', + solution_version='12.00', + project_version='14.0', + flat_sln=False, + uses_vcxproj=True, + path=path, + sdk_based=sdk_based, + default_toolset='v140'), '2013': VisualStudioVersion('2013', 'Visual Studio 2013', solution_version='13.00', @@ -316,7 +344,8 @@ def _DetectVisualStudioVersions(versions_to_check, force_express): 2008(e) - Visual Studio 2008 (9) 2010(e) - Visual Studio 2010 (10) 2012(e) - Visual Studio 2012 (11) - 2013(e) - Visual Studio 2013 (11) + 2013(e) - Visual Studio 2013 (12) + 2015 - Visual Studio 2015 (14) Where (e) is e for express editions of MSVS and blank otherwise. """ version_to_year = { @@ -325,6 +354,7 @@ def _DetectVisualStudioVersions(versions_to_check, force_express): '10.0': '2010', '11.0': '2012', '12.0': '2013', + '14.0': '2015', } versions = [] for version in versions_to_check: @@ -361,13 +391,14 @@ def _DetectVisualStudioVersions(versions_to_check, force_express): if not path: continue path = _ConvertToCygpath(path) - versions.append(_CreateVersion(version_to_year[version] + 'e', - os.path.join(path, '..'), sdk_based=True)) + if version != '14.0': # There is no Express edition for 2015. + versions.append(_CreateVersion(version_to_year[version] + 'e', + os.path.join(path, '..'), sdk_based=True)) return versions -def SelectVisualStudioVersion(version='auto'): +def SelectVisualStudioVersion(version='auto', allow_fallback=True): """Select which version of Visual Studio projects to generate. Arguments: @@ -379,7 +410,7 @@ def SelectVisualStudioVersion(version='auto'): if version == 'auto': version = os.environ.get('GYP_MSVS_VERSION', 'auto') version_map = { - 'auto': ('10.0', '12.0', '9.0', '8.0', '11.0'), + 'auto': ('14.0', '12.0', '10.0', '9.0', '8.0', '11.0'), '2005': ('8.0',), '2005e': ('8.0',), '2008': ('9.0',), @@ -390,6 +421,7 @@ def SelectVisualStudioVersion(version='auto'): '2012e': ('11.0',), '2013': ('12.0',), '2013e': ('12.0',), + '2015': ('14.0',), } override_path = os.environ.get('GYP_MSVS_OVERRIDE_PATH') if override_path: @@ -401,6 +433,8 @@ def SelectVisualStudioVersion(version='auto'): version = str(version) versions = _DetectVisualStudioVersions(version_map[version], 'e' in version) if not versions: + if not allow_fallback: + raise ValueError('Could not locate Visual Studio installation.') if version == 'auto': # Default to 2005 if we couldn't find anything return _CreateVersion('2005', None) diff --git a/tools/gyp/pylib/gyp/__init__.py b/tools/gyp/pylib/gyp/__init__.py index 30edea56752a..668f38b60d00 100755 --- a/tools/gyp/pylib/gyp/__init__.py +++ b/tools/gyp/pylib/gyp/__init__.py @@ -49,7 +49,7 @@ def FindBuildFiles(): def Load(build_files, format, default_variables={}, includes=[], depth='.', params=None, check=False, - circular_check=True): + circular_check=True, duplicate_basename_check=True): """ Loads one or more specified build files. default_variables and includes will be copied before use. @@ -59,7 +59,6 @@ def Load(build_files, format, default_variables={}, if params is None: params = {} - flavor = None if '-' in format: format, params['flavor'] = format.split('-', 1) @@ -69,6 +68,7 @@ def Load(build_files, format, default_variables={}, # named WITH_CAPITAL_LETTERS to provide a distinct "best practice" namespace, # avoiding collisions with user and automatic variables. default_variables['GENERATOR'] = format + default_variables['GENERATOR_FLAVOR'] = params.get('flavor', '') # Format can be a custom python file, or by default the name of a module # within gyp.generator. @@ -126,6 +126,7 @@ def Load(build_files, format, default_variables={}, # Process the input specific to this generator. result = gyp.input.Load(build_files, default_variables, includes[:], depth, generator_input_info, check, circular_check, + duplicate_basename_check, params['parallel'], params['root_targets']) return [generator] + result @@ -324,6 +325,16 @@ def gyp_main(args): parser.add_option('--no-circular-check', dest='circular_check', action='store_false', default=True, regenerate=False, help="don't check for circular relationships between files") + # --no-duplicate-basename-check disables the check for duplicate basenames + # in a static_library/shared_library project. Visual C++ 2008 generator + # doesn't support this configuration. Libtool on Mac also generates warnings + # when duplicate basenames are passed into Make generator on Mac. + # TODO(yukawa): Remove this option when these legacy generators are + # deprecated. + parser.add_option('--no-duplicate-basename-check', + dest='duplicate_basename_check', action='store_false', + default=True, regenerate=False, + help="don't check for duplicate basenames") parser.add_option('--no-parallel', action='store_true', default=False, help='Disable multiprocessing') parser.add_option('-S', '--suffix', dest='suffix', default='', @@ -371,7 +382,7 @@ def gyp_main(args): if options.use_environment: generate_formats = os.environ.get('GYP_GENERATORS', []) if generate_formats: - generate_formats = re.split('[\s,]', generate_formats) + generate_formats = re.split(r'[\s,]', generate_formats) if generate_formats: options.formats = generate_formats else: @@ -493,14 +504,14 @@ def gyp_main(args): 'gyp_binary': sys.argv[0], 'home_dot_gyp': home_dot_gyp, 'parallel': options.parallel, - 'root_targets': options.root_targets} + 'root_targets': options.root_targets, + 'target_arch': cmdline_default_variables.get('target_arch', '')} # Start with the default variables from the command line. - [generator, flat_list, targets, data] = Load(build_files, format, - cmdline_default_variables, - includes, options.depth, - params, options.check, - options.circular_check) + [generator, flat_list, targets, data] = Load( + build_files, format, cmdline_default_variables, includes, options.depth, + params, options.check, options.circular_check, + options.duplicate_basename_check) # TODO(mark): Pass |data| for now because the generator needs a list of # build files that came in. In the future, maybe it should just accept diff --git a/tools/gyp/pylib/gyp/common.py b/tools/gyp/pylib/gyp/common.py index f9c6c6f3a8cc..b6875e43efcb 100644 --- a/tools/gyp/pylib/gyp/common.py +++ b/tools/gyp/pylib/gyp/common.py @@ -4,6 +4,7 @@ from __future__ import with_statement +import collections import errno import filecmp import os.path @@ -328,7 +329,7 @@ def WriteOnDiff(filename): the target if it differs (on close). """ - class Writer: + class Writer(object): """Wrapper around file which only covers the target if it differs.""" def __init__(self): # Pick temporary file. @@ -472,6 +473,72 @@ def uniquer(seq, idfun=None): return result +# Based on http://code.activestate.com/recipes/576694/. +class OrderedSet(collections.MutableSet): + def __init__(self, iterable=None): + self.end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.map = {} # key --> [key, prev, next] + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.map) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + if key not in self.map: + end = self.end + curr = end[1] + curr[2] = end[1] = self.map[key] = [key, curr, end] + + def discard(self, key): + if key in self.map: + key, prev_item, next_item = self.map.pop(key) + prev_item[2] = next_item + next_item[1] = prev_item + + def __iter__(self): + end = self.end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + # The second argument is an addition that causes a pylint warning. + def pop(self, last=True): # pylint: disable=W0221 + if not self: + raise KeyError('set is empty') + key = self.end[1][0] if last else self.end[2][0] + self.discard(key) + return key + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self)) + + def __eq__(self, other): + if isinstance(other, OrderedSet): + return len(self) == len(other) and list(self) == list(other) + return set(self) == set(other) + + # Extensions to the recipe. + def update(self, iterable): + for i in iterable: + if i not in self: + self.add(i) + + class CycleError(Exception): """An exception raised when an unexpected cycle is detected.""" def __init__(self, nodes): @@ -481,7 +548,7 @@ def __str__(self): def TopologicallySorted(graph, get_edges): - """Topologically sort based on a user provided edge definition. + r"""Topologically sort based on a user provided edge definition. Args: graph: A list of node names. @@ -519,3 +586,14 @@ def Visit(node): for node in sorted(graph): Visit(node) return ordered_nodes + +def CrossCompileRequested(): + # TODO: figure out how to not build extra host objects in the + # non-cross-compile case when this is enabled, and enable unconditionally. + return (os.environ.get('GYP_CROSSCOMPILE') or + os.environ.get('AR_host') or + os.environ.get('CC_host') or + os.environ.get('CXX_host') or + os.environ.get('AR_target') or + os.environ.get('CC_target') or + os.environ.get('CXX_target')) diff --git a/tools/gyp/pylib/gyp/flock_tool.py b/tools/gyp/pylib/gyp/flock_tool.py index 3e7efff26ea8..b38d8660f728 100755 --- a/tools/gyp/pylib/gyp/flock_tool.py +++ b/tools/gyp/pylib/gyp/flock_tool.py @@ -40,7 +40,12 @@ def ExecFlock(self, lockfile, *cmd_list): # with EBADF, that's why we use this F_SETLK # hack instead. fd = os.open(lockfile, os.O_WRONLY|os.O_NOCTTY|os.O_CREAT, 0666) - op = struct.pack('hhllhhl', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) + if sys.platform.startswith('aix'): + # Python on AIX is compiled with LARGEFILE support, which changes the + # struct size. + op = struct.pack('hhIllqq', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) + else: + op = struct.pack('hhllhhl', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0) fcntl.fcntl(fd, fcntl.F_SETLK, op) return subprocess.call(cmd_list) diff --git a/tools/gyp/pylib/gyp/generator/analyzer.py b/tools/gyp/pylib/gyp/generator/analyzer.py new file mode 100644 index 000000000000..3a0ec9baff27 --- /dev/null +++ b/tools/gyp/pylib/gyp/generator/analyzer.py @@ -0,0 +1,608 @@ +# Copyright (c) 2014 Google Inc. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +This script is intended for use as a GYP_GENERATOR. It takes as input (by way of +the generator flag config_path) the path of a json file that dictates the files +and targets to search for. The following keys are supported: +files: list of paths (relative) of the files to search for. +targets: list of targets to search for. The target names are unqualified. + +The following is output: +error: only supplied if there is an error. +targets: the set of targets passed in via targets that either directly or + indirectly depend upon the set of paths supplied in files. +build_targets: minimal set of targets that directly depend on the changed + files and need to be built. The expectation is this set of targets is passed + into a build step. +status: outputs one of three values: none of the supplied files were found, + one of the include files changed so that it should be assumed everything + changed (in this case targets and build_targets are not output) or at + least one file was found. +invalid_targets: list of supplied targets thare were not found. + +If the generator flag analyzer_output_path is specified, output is written +there. Otherwise output is written to stdout. +""" + +import gyp.common +import gyp.ninja_syntax as ninja_syntax +import json +import os +import posixpath +import sys + +debug = False + +found_dependency_string = 'Found dependency' +no_dependency_string = 'No dependencies' +# Status when it should be assumed that everything has changed. +all_changed_string = 'Found dependency (all)' + +# MatchStatus is used indicate if and how a target depends upon the supplied +# sources. +# The target's sources contain one of the supplied paths. +MATCH_STATUS_MATCHES = 1 +# The target has a dependency on another target that contains one of the +# supplied paths. +MATCH_STATUS_MATCHES_BY_DEPENDENCY = 2 +# The target's sources weren't in the supplied paths and none of the target's +# dependencies depend upon a target that matched. +MATCH_STATUS_DOESNT_MATCH = 3 +# The target doesn't contain the source, but the dependent targets have not yet +# been visited to determine a more specific status yet. +MATCH_STATUS_TBD = 4 + +generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested() + +generator_wants_static_library_dependencies_adjusted = False + +generator_default_variables = { +} +for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR', + 'LIB_DIR', 'SHARED_LIB_DIR']: + generator_default_variables[dirname] = '!!!' + +for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME', + 'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT', + 'EXECUTABLE_PREFIX', 'EXECUTABLE_SUFFIX', + 'STATIC_LIB_PREFIX', 'STATIC_LIB_SUFFIX', + 'SHARED_LIB_PREFIX', 'SHARED_LIB_SUFFIX', + 'CONFIGURATION_NAME']: + generator_default_variables[unused] = '' + + +def _ToGypPath(path): + """Converts a path to the format used by gyp.""" + if os.sep == '\\' and os.altsep == '/': + return path.replace('\\', '/') + return path + + +def _ResolveParent(path, base_path_components): + """Resolves |path|, which starts with at least one '../'. Returns an empty + string if the path shouldn't be considered. See _AddSources() for a + description of |base_path_components|.""" + depth = 0 + while path.startswith('../'): + depth += 1 + path = path[3:] + # Relative includes may go outside the source tree. For example, an action may + # have inputs in /usr/include, which are not in the source tree. + if depth > len(base_path_components): + return '' + if depth == len(base_path_components): + return path + return '/'.join(base_path_components[0:len(base_path_components) - depth]) + \ + '/' + path + + +def _AddSources(sources, base_path, base_path_components, result): + """Extracts valid sources from |sources| and adds them to |result|. Each + source file is relative to |base_path|, but may contain '..'. To make + resolving '..' easier |base_path_components| contains each of the + directories in |base_path|. Additionally each source may contain variables. + Such sources are ignored as it is assumed dependencies on them are expressed + and tracked in some other means.""" + # NOTE: gyp paths are always posix style. + for source in sources: + if not len(source) or source.startswith('!!!') or source.startswith('$'): + continue + # variable expansion may lead to //. + org_source = source + source = source[0] + source[1:].replace('//', '/') + if source.startswith('../'): + source = _ResolveParent(source, base_path_components) + if len(source): + result.append(source) + continue + result.append(base_path + source) + if debug: + print 'AddSource', org_source, result[len(result) - 1] + + +def _ExtractSourcesFromAction(action, base_path, base_path_components, + results): + if 'inputs' in action: + _AddSources(action['inputs'], base_path, base_path_components, results) + + +def _ToLocalPath(toplevel_dir, path): + """Converts |path| to a path relative to |toplevel_dir|.""" + if path == toplevel_dir: + return '' + if path.startswith(toplevel_dir + '/'): + return path[len(toplevel_dir) + len('/'):] + return path + + +def _ExtractSources(target, target_dict, toplevel_dir): + # |target| is either absolute or relative and in the format of the OS. Gyp + # source paths are always posix. Convert |target| to a posix path relative to + # |toplevel_dir_|. This is done to make it easy to build source paths. + base_path = posixpath.dirname(_ToLocalPath(toplevel_dir, _ToGypPath(target))) + base_path_components = base_path.split('/') + + # Add a trailing '/' so that _AddSources() can easily build paths. + if len(base_path): + base_path += '/' + + if debug: + print 'ExtractSources', target, base_path + + results = [] + if 'sources' in target_dict: + _AddSources(target_dict['sources'], base_path, base_path_components, + results) + # Include the inputs from any actions. Any changes to these affect the + # resulting output. + if 'actions' in target_dict: + for action in target_dict['actions']: + _ExtractSourcesFromAction(action, base_path, base_path_components, + results) + if 'rules' in target_dict: + for rule in target_dict['rules']: + _ExtractSourcesFromAction(rule, base_path, base_path_components, results) + + return results + + +class Target(object): + """Holds information about a particular target: + deps: set of Targets this Target depends upon. This is not recursive, only the + direct dependent Targets. + match_status: one of the MatchStatus values. + back_deps: set of Targets that have a dependency on this Target. + visited: used during iteration to indicate whether we've visited this target. + This is used for two iterations, once in building the set of Targets and + again in _GetBuildTargets(). + name: fully qualified name of the target. + requires_build: True if the target type is such that it needs to be built. + See _DoesTargetTypeRequireBuild for details. + added_to_compile_targets: used when determining if the target was added to the + set of targets that needs to be built. + in_roots: true if this target is a descendant of one of the root nodes. + is_executable: true if the type of target is executable. + is_static_library: true if the type of target is static_library. + is_or_has_linked_ancestor: true if the target does a link (eg executable), or + if there is a target in back_deps that does a link.""" + def __init__(self, name): + self.deps = set() + self.match_status = MATCH_STATUS_TBD + self.back_deps = set() + self.name = name + # TODO(sky): I don't like hanging this off Target. This state is specific + # to certain functions and should be isolated there. + self.visited = False + self.requires_build = False + self.added_to_compile_targets = False + self.in_roots = False + self.is_executable = False + self.is_static_library = False + self.is_or_has_linked_ancestor = False + + +class Config(object): + """Details what we're looking for + files: set of files to search for + targets: see file description for details.""" + def __init__(self): + self.files = [] + self.targets = set() + + def Init(self, params): + """Initializes Config. This is a separate method as it raises an exception + if there is a parse error.""" + generator_flags = params.get('generator_flags', {}) + config_path = generator_flags.get('config_path', None) + if not config_path: + return + try: + f = open(config_path, 'r') + config = json.load(f) + f.close() + except IOError: + raise Exception('Unable to open file ' + config_path) + except ValueError as e: + raise Exception('Unable to parse config file ' + config_path + str(e)) + if not isinstance(config, dict): + raise Exception('config_path must be a JSON file containing a dictionary') + self.files = config.get('files', []) + self.targets = set(config.get('targets', [])) + + +def _WasBuildFileModified(build_file, data, files, toplevel_dir): + """Returns true if the build file |build_file| is either in |files| or + one of the files included by |build_file| is in |files|. |toplevel_dir| is + the root of the source tree.""" + if _ToLocalPath(toplevel_dir, _ToGypPath(build_file)) in files: + if debug: + print 'gyp file modified', build_file + return True + + # First element of included_files is the file itself. + if len(data[build_file]['included_files']) <= 1: + return False + + for include_file in data[build_file]['included_files'][1:]: + # |included_files| are relative to the directory of the |build_file|. + rel_include_file = \ + _ToGypPath(gyp.common.UnrelativePath(include_file, build_file)) + if _ToLocalPath(toplevel_dir, rel_include_file) in files: + if debug: + print 'included gyp file modified, gyp_file=', build_file, \ + 'included file=', rel_include_file + return True + return False + + +def _GetOrCreateTargetByName(targets, target_name): + """Creates or returns the Target at targets[target_name]. If there is no + Target for |target_name| one is created. Returns a tuple of whether a new + Target was created and the Target.""" + if target_name in targets: + return False, targets[target_name] + target = Target(target_name) + targets[target_name] = target + return True, target + + +def _DoesTargetTypeRequireBuild(target_dict): + """Returns true if the target type is such that it needs to be built.""" + # If a 'none' target has rules or actions we assume it requires a build. + return bool(target_dict['type'] != 'none' or + target_dict.get('actions') or target_dict.get('rules')) + + +def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files, + build_files): + """Returns a tuple of the following: + . A dictionary mapping from fully qualified name to Target. + . A list of the targets that have a source file in |files|. + . Set of root Targets reachable from the the files |build_files|. + This sets the |match_status| of the targets that contain any of the source + files in |files| to MATCH_STATUS_MATCHES. + |toplevel_dir| is the root of the source tree.""" + # Maps from target name to Target. + targets = {} + + # Targets that matched. + matching_targets = [] + + # Queue of targets to visit. + targets_to_visit = target_list[:] + + # Maps from build file to a boolean indicating whether the build file is in + # |files|. + build_file_in_files = {} + + # Root targets across all files. + roots = set() + + # Set of Targets in |build_files|. + build_file_targets = set() + + while len(targets_to_visit) > 0: + target_name = targets_to_visit.pop() + created_target, target = _GetOrCreateTargetByName(targets, target_name) + if created_target: + roots.add(target) + elif target.visited: + continue + + target.visited = True + target.requires_build = _DoesTargetTypeRequireBuild( + target_dicts[target_name]) + target_type = target_dicts[target_name]['type'] + target.is_executable = target_type == 'executable' + target.is_static_library = target_type == 'static_library' + target.is_or_has_linked_ancestor = (target_type == 'executable' or + target_type == 'shared_library') + + build_file = gyp.common.ParseQualifiedTarget(target_name)[0] + if not build_file in build_file_in_files: + build_file_in_files[build_file] = \ + _WasBuildFileModified(build_file, data, files, toplevel_dir) + + if build_file in build_files: + build_file_targets.add(target) + + # If a build file (or any of its included files) is modified we assume all + # targets in the file are modified. + if build_file_in_files[build_file]: + print 'matching target from modified build file', target_name + target.match_status = MATCH_STATUS_MATCHES + matching_targets.append(target) + else: + sources = _ExtractSources(target_name, target_dicts[target_name], + toplevel_dir) + for source in sources: + if source in files: + print 'target', target_name, 'matches', source + target.match_status = MATCH_STATUS_MATCHES + matching_targets.append(target) + break + + # Add dependencies to visit as well as updating back pointers for deps. + for dep in target_dicts[target_name].get('dependencies', []): + targets_to_visit.append(dep) + + created_dep_target, dep_target = _GetOrCreateTargetByName(targets, dep) + if not created_dep_target: + roots.discard(dep_target) + + target.deps.add(dep_target) + dep_target.back_deps.add(target) + + return targets, matching_targets, roots & build_file_targets + + +def _GetUnqualifiedToTargetMapping(all_targets, to_find): + """Returns a mapping (dictionary) from unqualified name to Target for all the + Targets in |to_find|.""" + result = {} + if not to_find: + return result + to_find = set(to_find) + for target_name in all_targets.keys(): + extracted = gyp.common.ParseQualifiedTarget(target_name) + if len(extracted) > 1 and extracted[1] in to_find: + to_find.remove(extracted[1]) + result[extracted[1]] = all_targets[target_name] + if not to_find: + return result + return result + + +def _DoesTargetDependOn(target): + """Returns true if |target| or any of its dependencies matches the supplied + set of paths. This updates |matches| of the Targets as it recurses. + target: the Target to look for.""" + if target.match_status == MATCH_STATUS_DOESNT_MATCH: + return False + if target.match_status == MATCH_STATUS_MATCHES or \ + target.match_status == MATCH_STATUS_MATCHES_BY_DEPENDENCY: + return True + for dep in target.deps: + if _DoesTargetDependOn(dep): + target.match_status = MATCH_STATUS_MATCHES_BY_DEPENDENCY + print '\t', target.name, 'matches by dep', dep.name + return True + target.match_status = MATCH_STATUS_DOESNT_MATCH + return False + + +def _GetTargetsDependingOn(possible_targets): + """Returns the list of Targets in |possible_targets| that depend (either + directly on indirectly) on the matched targets. + possible_targets: targets to search from.""" + found = [] + print 'Targets that matched by dependency:' + for target in possible_targets: + if _DoesTargetDependOn(target): + found.append(target) + return found + + +def _AddBuildTargets(target, roots, add_if_no_ancestor, result): + """Recurses through all targets that depend on |target|, adding all targets + that need to be built (and are in |roots|) to |result|. + roots: set of root targets. + add_if_no_ancestor: If true and there are no ancestors of |target| then add + |target| to |result|. |target| must still be in |roots|. + result: targets that need to be built are added here.""" + if target.visited: + return + + target.visited = True + target.in_roots = not target.back_deps and target in roots + + for back_dep_target in target.back_deps: + _AddBuildTargets(back_dep_target, roots, False, result) + target.added_to_compile_targets |= back_dep_target.added_to_compile_targets + target.in_roots |= back_dep_target.in_roots + target.is_or_has_linked_ancestor |= ( + back_dep_target.is_or_has_linked_ancestor) + + # Always add 'executable' targets. Even though they may be built by other + # targets that depend upon them it makes detection of what is going to be + # built easier. + # And always add static_libraries that have no dependencies on them from + # linkables. This is necessary as the other dependencies on them may be + # static libraries themselves, which are not compile time dependencies. + if target.in_roots and \ + (target.is_executable or + (not target.added_to_compile_targets and + (add_if_no_ancestor or target.requires_build)) or + (target.is_static_library and add_if_no_ancestor and + not target.is_or_has_linked_ancestor)): + print '\t\tadding to build targets', target.name, 'executable', \ + target.is_executable, 'added_to_compile_targets', \ + target.added_to_compile_targets, 'add_if_no_ancestor', \ + add_if_no_ancestor, 'requires_build', target.requires_build, \ + 'is_static_library', target.is_static_library, \ + 'is_or_has_linked_ancestor', target.is_or_has_linked_ancestor + result.add(target) + target.added_to_compile_targets = True + + +def _GetBuildTargets(matching_targets, roots): + """Returns the set of Targets that require a build. + matching_targets: targets that changed and need to be built. + roots: set of root targets in the build files to search from.""" + result = set() + for target in matching_targets: + print '\tfinding build targets for match', target.name + _AddBuildTargets(target, roots, True, result) + return result + + +def _WriteOutput(params, **values): + """Writes the output, either to stdout or a file is specified.""" + if 'error' in values: + print 'Error:', values['error'] + if 'status' in values: + print values['status'] + if 'targets' in values: + values['targets'].sort() + print 'Supplied targets that depend on changed files:' + for target in values['targets']: + print '\t', target + if 'invalid_targets' in values: + values['invalid_targets'].sort() + print 'The following targets were not found:' + for target in values['invalid_targets']: + print '\t', target + if 'build_targets' in values: + values['build_targets'].sort() + print 'Targets that require a build:' + for target in values['build_targets']: + print '\t', target + + output_path = params.get('generator_flags', {}).get( + 'analyzer_output_path', None) + if not output_path: + print json.dumps(values) + return + try: + f = open(output_path, 'w') + f.write(json.dumps(values) + '\n') + f.close() + except IOError as e: + print 'Error writing to output file', output_path, str(e) + + +def _WasGypIncludeFileModified(params, files): + """Returns true if one of the files in |files| is in the set of included + files.""" + if params['options'].includes: + for include in params['options'].includes: + if _ToGypPath(include) in files: + print 'Include file modified, assuming all changed', include + return True + return False + + +def _NamesNotIn(names, mapping): + """Returns a list of the values in |names| that are not in |mapping|.""" + return [name for name in names if name not in mapping] + + +def _LookupTargets(names, mapping): + """Returns a list of the mapping[name] for each value in |names| that is in + |mapping|.""" + return [mapping[name] for name in names if name in mapping] + + +def CalculateVariables(default_variables, params): + """Calculate additional variables for use in the build (called by gyp).""" + flavor = gyp.common.GetFlavor(params) + if flavor == 'mac': + default_variables.setdefault('OS', 'mac') + elif flavor == 'win': + default_variables.setdefault('OS', 'win') + # Copy additional generator configuration data from VS, which is shared + # by the Windows Ninja generator. + import gyp.generator.msvs as msvs_generator + generator_additional_non_configuration_keys = getattr(msvs_generator, + 'generator_additional_non_configuration_keys', []) + generator_additional_path_sections = getattr(msvs_generator, + 'generator_additional_path_sections', []) + + gyp.msvs_emulation.CalculateCommonVariables(default_variables, params) + else: + operating_system = flavor + if flavor == 'android': + operating_system = 'linux' # Keep this legacy behavior for now. + default_variables.setdefault('OS', operating_system) + + +def GenerateOutput(target_list, target_dicts, data, params): + """Called by gyp as the final stage. Outputs results.""" + config = Config() + try: + config.Init(params) + if not config.files: + raise Exception('Must specify files to analyze via config_path generator ' + 'flag') + + toplevel_dir = _ToGypPath(os.path.abspath(params['options'].toplevel_dir)) + if debug: + print 'toplevel_dir', toplevel_dir + + if _WasGypIncludeFileModified(params, config.files): + result_dict = { 'status': all_changed_string, + 'targets': list(config.targets) } + _WriteOutput(params, **result_dict) + return + + all_targets, matching_targets, roots = _GenerateTargets( + data, target_list, target_dicts, toplevel_dir, frozenset(config.files), + params['build_files']) + + print 'roots:' + for root in roots: + print '\t', root.name + + unqualified_mapping = _GetUnqualifiedToTargetMapping(all_targets, + config.targets) + invalid_targets = None + if len(unqualified_mapping) != len(config.targets): + invalid_targets = _NamesNotIn(config.targets, unqualified_mapping) + + if matching_targets: + search_targets = _LookupTargets(config.targets, unqualified_mapping) + print 'supplied targets' + for target in config.targets: + print '\t', target + print 'expanded supplied targets' + for target in search_targets: + print '\t', target.name + matched_search_targets = _GetTargetsDependingOn(search_targets) + print 'raw matched search targets:' + for target in matched_search_targets: + print '\t', target.name + # Reset the visited status for _GetBuildTargets. + for target in all_targets.itervalues(): + target.visited = False + print 'Finding build targets' + build_targets = _GetBuildTargets(matching_targets, roots) + matched_search_targets = [gyp.common.ParseQualifiedTarget(target.name)[1] + for target in matched_search_targets] + build_targets = [gyp.common.ParseQualifiedTarget(target.name)[1] + for target in build_targets] + else: + matched_search_targets = [] + build_targets = [] + + result_dict = { 'targets': matched_search_targets, + 'status': found_dependency_string if matching_targets else + no_dependency_string, + 'build_targets': build_targets} + if invalid_targets: + result_dict['invalid_targets'] = invalid_targets + _WriteOutput(params, **result_dict) + + except Exception as e: + _WriteOutput(params, error=str(e)) diff --git a/tools/gyp/pylib/gyp/generator/android.py b/tools/gyp/pylib/gyp/generator/android.py deleted file mode 100644 index 41346e2b1c35..000000000000 --- a/tools/gyp/pylib/gyp/generator/android.py +++ /dev/null @@ -1,1069 +0,0 @@ -# Copyright (c) 2012 Google Inc. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Notes: -# -# This generates makefiles suitable for inclusion into the Android build system -# via an Android.mk file. It is based on make.py, the standard makefile -# generator. -# -# The code below generates a separate .mk file for each target, but -# all are sourced by the top-level GypAndroid.mk. This means that all -# variables in .mk-files clobber one another, and furthermore that any -# variables set potentially clash with other Android build system variables. -# Try to avoid setting global variables where possible. - -import gyp -import gyp.common -import gyp.generator.make as make # Reuse global functions from make backend. -import os -import re -import subprocess - -generator_default_variables = { - 'OS': 'android', - 'EXECUTABLE_PREFIX': '', - 'EXECUTABLE_SUFFIX': '', - 'STATIC_LIB_PREFIX': 'lib', - 'SHARED_LIB_PREFIX': 'lib', - 'STATIC_LIB_SUFFIX': '.a', - 'SHARED_LIB_SUFFIX': '.so', - 'INTERMEDIATE_DIR': '$(gyp_intermediate_dir)', - 'SHARED_INTERMEDIATE_DIR': '$(gyp_shared_intermediate_dir)', - 'PRODUCT_DIR': '$(gyp_shared_intermediate_dir)', - 'SHARED_LIB_DIR': '$(builddir)/lib.$(TOOLSET)', - 'LIB_DIR': '$(obj).$(TOOLSET)', - 'RULE_INPUT_ROOT': '%(INPUT_ROOT)s', # This gets expanded by Python. - 'RULE_INPUT_DIRNAME': '%(INPUT_DIRNAME)s', # This gets expanded by Python. - 'RULE_INPUT_PATH': '$(RULE_SOURCES)', - 'RULE_INPUT_EXT': '$(suffix $<)', - 'RULE_INPUT_NAME': '$(notdir $<)', - 'CONFIGURATION_NAME': '$(GYP_CONFIGURATION)', -} - -# Make supports multiple toolsets -generator_supports_multiple_toolsets = True - - -# Generator-specific gyp specs. -generator_additional_non_configuration_keys = [ - # Boolean to declare that this target does not want its name mangled. - 'android_unmangled_name', -] -generator_additional_path_sections = [] -generator_extra_sources_for_rules = [] - - -SHARED_FOOTER = """\ -# "gyp_all_modules" is a concatenation of the "gyp_all_modules" targets from -# all the included sub-makefiles. This is just here to clarify. -gyp_all_modules: -""" - -header = """\ -# This file is generated by gyp; do not edit. - -""" - -android_standard_include_paths = set([ - # JNI_H_INCLUDE in build/core/binary.mk - 'dalvik/libnativehelper/include/nativehelper', - # from SRC_HEADERS in build/core/config.mk - 'system/core/include', - 'hardware/libhardware/include', - 'hardware/libhardware_legacy/include', - 'hardware/ril/include', - 'dalvik/libnativehelper/include', - 'frameworks/native/include', - 'frameworks/native/opengl/include', - 'frameworks/base/include', - 'frameworks/base/opengl/include', - 'frameworks/base/native/include', - 'external/skia/include', - # TARGET_C_INCLUDES in build/core/combo/TARGET_linux-arm.mk - 'bionic/libc/arch-arm/include', - 'bionic/libc/include', - 'bionic/libstdc++/include', - 'bionic/libc/kernel/common', - 'bionic/libc/kernel/arch-arm', - 'bionic/libm/include', - 'bionic/libm/include/arm', - 'bionic/libthread_db/include', - ]) - - -# Map gyp target types to Android module classes. -MODULE_CLASSES = { - 'static_library': 'STATIC_LIBRARIES', - 'shared_library': 'SHARED_LIBRARIES', - 'executable': 'EXECUTABLES', -} - - -def IsCPPExtension(ext): - return make.COMPILABLE_EXTENSIONS.get(ext) == 'cxx' - - -def Sourceify(path): - """Convert a path to its source directory form. The Android backend does not - support options.generator_output, so this function is a noop.""" - return path - - -# Map from qualified target to path to output. -# For Android, the target of these maps is a tuple ('static', 'modulename'), -# ('dynamic', 'modulename'), or ('path', 'some/path') instead of a string, -# since we link by module. -target_outputs = {} -# Map from qualified target to any linkable output. A subset -# of target_outputs. E.g. when mybinary depends on liba, we want to -# include liba in the linker line; when otherbinary depends on -# mybinary, we just want to build mybinary first. -target_link_deps = {} - - -class AndroidMkWriter(object): - """AndroidMkWriter packages up the writing of one target-specific Android.mk. - - Its only real entry point is Write(), and is mostly used for namespacing. - """ - - def __init__(self, android_top_dir): - self.android_top_dir = android_top_dir - - def Write(self, qualified_target, relative_target, base_path, output_filename, - spec, configs, part_of_all): - """The main entry point: writes a .mk file for a single target. - - Arguments: - qualified_target: target we're generating - relative_target: qualified target name relative to the root - base_path: path relative to source root we're building in, used to resolve - target-relative paths - output_filename: output .mk file name to write - spec, configs: gyp info - part_of_all: flag indicating this target is part of 'all' - """ - gyp.common.EnsureDirExists(output_filename) - - self.fp = open(output_filename, 'w') - - self.fp.write(header) - - self.qualified_target = qualified_target - self.relative_target = relative_target - self.path = base_path - self.target = spec['target_name'] - self.type = spec['type'] - self.toolset = spec['toolset'] - - deps, link_deps = self.ComputeDeps(spec) - - # Some of the generation below can add extra output, sources, or - # link dependencies. All of the out params of the functions that - # follow use names like extra_foo. - extra_outputs = [] - extra_sources = [] - - self.android_class = MODULE_CLASSES.get(self.type, 'GYP') - self.android_module = self.ComputeAndroidModule(spec) - (self.android_stem, self.android_suffix) = self.ComputeOutputParts(spec) - self.output = self.output_binary = self.ComputeOutput(spec) - - # Standard header. - self.WriteLn('include $(CLEAR_VARS)\n') - - # Module class and name. - self.WriteLn('LOCAL_MODULE_CLASS := ' + self.android_class) - self.WriteLn('LOCAL_MODULE := ' + self.android_module) - # Only emit LOCAL_MODULE_STEM if it's different to LOCAL_MODULE. - # The library module classes fail if the stem is set. ComputeOutputParts - # makes sure that stem == modulename in these cases. - if self.android_stem != self.android_module: - self.WriteLn('LOCAL_MODULE_STEM := ' + self.android_stem) - self.WriteLn('LOCAL_MODULE_SUFFIX := ' + self.android_suffix) - self.WriteLn('LOCAL_MODULE_TAGS := optional') - if self.toolset == 'host': - self.WriteLn('LOCAL_IS_HOST_MODULE := true') - - # Grab output directories; needed for Actions and Rules. - self.WriteLn('gyp_intermediate_dir := $(call local-intermediates-dir)') - self.WriteLn('gyp_shared_intermediate_dir := ' - '$(call intermediates-dir-for,GYP,shared)') - self.WriteLn() - - # List files this target depends on so that actions/rules/copies/sources - # can depend on the list. - # TODO: doesn't pull in things through transitive link deps; needed? - target_dependencies = [x[1] for x in deps if x[0] == 'path'] - self.WriteLn('# Make sure our deps are built first.') - self.WriteList(target_dependencies, 'GYP_TARGET_DEPENDENCIES', - local_pathify=True) - - # Actions must come first, since they can generate more OBJs for use below. - if 'actions' in spec: - self.WriteActions(spec['actions'], extra_sources, extra_outputs) - - # Rules must be early like actions. - if 'rules' in spec: - self.WriteRules(spec['rules'], extra_sources, extra_outputs) - - if 'copies' in spec: - self.WriteCopies(spec['copies'], extra_outputs) - - # GYP generated outputs. - self.WriteList(extra_outputs, 'GYP_GENERATED_OUTPUTS', local_pathify=True) - - # Set LOCAL_ADDITIONAL_DEPENDENCIES so that Android's build rules depend - # on both our dependency targets and our generated files. - self.WriteLn('# Make sure our deps and generated files are built first.') - self.WriteLn('LOCAL_ADDITIONAL_DEPENDENCIES := $(GYP_TARGET_DEPENDENCIES) ' - '$(GYP_GENERATED_OUTPUTS)') - self.WriteLn() - - # Sources. - if spec.get('sources', []) or extra_sources: - self.WriteSources(spec, configs, extra_sources) - - self.WriteTarget(spec, configs, deps, link_deps, part_of_all) - - # Update global list of target outputs, used in dependency tracking. - target_outputs[qualified_target] = ('path', self.output_binary) - - # Update global list of link dependencies. - if self.type == 'static_library': - target_link_deps[qualified_target] = ('static', self.android_module) - elif self.type == 'shared_library': - target_link_deps[qualified_target] = ('shared', self.android_module) - - self.fp.close() - return self.android_module - - - def WriteActions(self, actions, extra_sources, extra_outputs): - """Write Makefile code for any 'actions' from the gyp input. - - extra_sources: a list that will be filled in with newly generated source - files, if any - extra_outputs: a list that will be filled in with any outputs of these - actions (used to make other pieces dependent on these - actions) - """ - for action in actions: - name = make.StringToMakefileVariable('%s_%s' % (self.relative_target, - action['action_name'])) - self.WriteLn('### Rules for action "%s":' % action['action_name']) - inputs = action['inputs'] - outputs = action['outputs'] - - # Build up a list of outputs. - # Collect the output dirs we'll need. - dirs = set() - for out in outputs: - if not out.startswith('$'): - print ('WARNING: Action for target "%s" writes output to local path ' - '"%s".' % (self.target, out)) - dir = os.path.split(out)[0] - if dir: - dirs.add(dir) - if int(action.get('process_outputs_as_sources', False)): - extra_sources += outputs - - # Prepare the actual command. - command = gyp.common.EncodePOSIXShellList(action['action']) - if 'message' in action: - quiet_cmd = 'Gyp action: %s ($@)' % action['message'] - else: - quiet_cmd = 'Gyp action: %s ($@)' % name - if len(dirs) > 0: - command = 'mkdir -p %s' % ' '.join(dirs) + '; ' + command - - cd_action = 'cd $(gyp_local_path)/%s; ' % self.path - command = cd_action + command - - # The makefile rules are all relative to the top dir, but the gyp actions - # are defined relative to their containing dir. This replaces the gyp_* - # variables for the action rule with an absolute version so that the - # output goes in the right place. - # Only write the gyp_* rules for the "primary" output (:1); - # it's superfluous for the "extra outputs", and this avoids accidentally - # writing duplicate dummy rules for those outputs. - main_output = make.QuoteSpaces(self.LocalPathify(outputs[0])) - self.WriteLn('%s: gyp_local_path := $(LOCAL_PATH)' % main_output) - self.WriteLn('%s: gyp_intermediate_dir := ' - '$(abspath $(gyp_intermediate_dir))' % main_output) - self.WriteLn('%s: gyp_shared_intermediate_dir := ' - '$(abspath $(gyp_shared_intermediate_dir))' % main_output) - - # Android's envsetup.sh adds a number of directories to the path including - # the built host binary directory. This causes actions/rules invoked by - # gyp to sometimes use these instead of system versions, e.g. bison. - # The built host binaries may not be suitable, and can cause errors. - # So, we remove them from the PATH using the ANDROID_BUILD_PATHS variable - # set by envsetup. - self.WriteLn('%s: export PATH := $(subst $(ANDROID_BUILD_PATHS),,$(PATH))' - % main_output) - - for input in inputs: - assert ' ' not in input, ( - "Spaces in action input filenames not supported (%s)" % input) - for output in outputs: - assert ' ' not in output, ( - "Spaces in action output filenames not supported (%s)" % output) - - self.WriteLn('%s: %s $(GYP_TARGET_DEPENDENCIES)' % - (main_output, ' '.join(map(self.LocalPathify, inputs)))) - self.WriteLn('\t@echo "%s"' % quiet_cmd) - self.WriteLn('\t$(hide)%s\n' % command) - for output in outputs[1:]: - # Make each output depend on the main output, with an empty command - # to force make to notice that the mtime has changed. - self.WriteLn('%s: %s ;' % (self.LocalPathify(output), main_output)) - - extra_outputs += outputs - self.WriteLn() - - self.WriteLn() - - - def WriteRules(self, rules, extra_sources, extra_outputs): - """Write Makefile code for any 'rules' from the gyp input. - - extra_sources: a list that will be filled in with newly generated source - files, if any - extra_outputs: a list that will be filled in with any outputs of these - rules (used to make other pieces dependent on these rules) - """ - if len(rules) == 0: - return - rule_trigger = '%s_rule_trigger' % self.android_module - - did_write_rule = False - for rule in rules: - if len(rule.get('rule_sources', [])) == 0: - continue - did_write_rule = True - name = make.StringToMakefileVariable('%s_%s' % (self.relative_target, - rule['rule_name'])) - self.WriteLn('\n### Generated for rule "%s":' % name) - self.WriteLn('# "%s":' % rule) - - inputs = rule.get('inputs') - for rule_source in rule.get('rule_sources', []): - (rule_source_dirname, rule_source_basename) = os.path.split(rule_source) - (rule_source_root, rule_source_ext) = \ - os.path.splitext(rule_source_basename) - - outputs = [self.ExpandInputRoot(out, rule_source_root, - rule_source_dirname) - for out in rule['outputs']] - - dirs = set() - for out in outputs: - if not out.startswith('$'): - print ('WARNING: Rule for target %s writes output to local path %s' - % (self.target, out)) - dir = os.path.dirname(out) - if dir: - dirs.add(dir) - extra_outputs += outputs - if int(rule.get('process_outputs_as_sources', False)): - extra_sources.extend(outputs) - - components = [] - for component in rule['action']: - component = self.ExpandInputRoot(component, rule_source_root, - rule_source_dirname) - if '$(RULE_SOURCES)' in component: - component = component.replace('$(RULE_SOURCES)', - rule_source) - components.append(component) - - command = gyp.common.EncodePOSIXShellList(components) - cd_action = 'cd $(gyp_local_path)/%s; ' % self.path - command = cd_action + command - if dirs: - command = 'mkdir -p %s' % ' '.join(dirs) + '; ' + command - - # We set up a rule to build the first output, and then set up - # a rule for each additional output to depend on the first. - outputs = map(self.LocalPathify, outputs) - main_output = outputs[0] - self.WriteLn('%s: gyp_local_path := $(LOCAL_PATH)' % main_output) - self.WriteLn('%s: gyp_intermediate_dir := ' - '$(abspath $(gyp_intermediate_dir))' % main_output) - self.WriteLn('%s: gyp_shared_intermediate_dir := ' - '$(abspath $(gyp_shared_intermediate_dir))' % main_output) - - # See explanation in WriteActions. - self.WriteLn('%s: export PATH := ' - '$(subst $(ANDROID_BUILD_PATHS),,$(PATH))' % main_output) - - main_output_deps = self.LocalPathify(rule_source) - if inputs: - main_output_deps += ' ' - main_output_deps += ' '.join([self.LocalPathify(f) for f in inputs]) - - self.WriteLn('%s: %s $(GYP_TARGET_DEPENDENCIES)' % - (main_output, main_output_deps)) - self.WriteLn('\t%s\n' % command) - for output in outputs[1:]: - # Make each output depend on the main output, with an empty command - # to force make to notice that the mtime has changed. - self.WriteLn('%s: %s ;' % (output, main_output)) - self.WriteLn('.PHONY: %s' % (rule_trigger)) - self.WriteLn('%s: %s' % (rule_trigger, main_output)) - self.WriteLn('') - if did_write_rule: - extra_sources.append(rule_trigger) # Force all rules to run. - self.WriteLn('### Finished generating for all rules') - self.WriteLn('') - - - def WriteCopies(self, copies, extra_outputs): - """Write Makefile code for any 'copies' from the gyp input. - - extra_outputs: a list that will be filled in with any outputs of this action - (used to make other pieces dependent on this action) - """ - self.WriteLn('### Generated for copy rule.') - - variable = make.StringToMakefileVariable(self.relative_target + '_copies') - outputs = [] - for copy in copies: - for path in copy['files']: - # The Android build system does not allow generation of files into the - # source tree. The destination should start with a variable, which will - # typically be $(gyp_intermediate_dir) or - # $(gyp_shared_intermediate_dir). Note that we can't use an assertion - # because some of the gyp tests depend on this. - if not copy['destination'].startswith('$'): - print ('WARNING: Copy rule for target %s writes output to ' - 'local path %s' % (self.target, copy['destination'])) - - # LocalPathify() calls normpath, stripping trailing slashes. - path = Sourceify(self.LocalPathify(path)) - filename = os.path.split(path)[1] - output = Sourceify(self.LocalPathify(os.path.join(copy['destination'], - filename))) - - self.WriteLn('%s: %s $(GYP_TARGET_DEPENDENCIES) | $(ACP)' % - (output, path)) - self.WriteLn('\t@echo Copying: $@') - self.WriteLn('\t$(hide) mkdir -p $(dir $@)') - self.WriteLn('\t$(hide) $(ACP) -rpf $< $@') - self.WriteLn() - outputs.append(output) - self.WriteLn('%s = %s' % (variable, - ' '.join(map(make.QuoteSpaces, outputs)))) - extra_outputs.append('$(%s)' % variable) - self.WriteLn() - - - def WriteSourceFlags(self, spec, configs): - """Write out the flags and include paths used to compile source files for - the current target. - - Args: - spec, configs: input from gyp. - """ - for configname, config in sorted(configs.iteritems()): - extracted_includes = [] - - self.WriteLn('\n# Flags passed to both C and C++ files.') - cflags, includes_from_cflags = self.ExtractIncludesFromCFlags( - config.get('cflags', []) + config.get('cflags_c', [])) - extracted_includes.extend(includes_from_cflags) - self.WriteList(cflags, 'MY_CFLAGS_%s' % configname) - - self.WriteList(config.get('defines'), 'MY_DEFS_%s' % configname, - prefix='-D', quoter=make.EscapeCppDefine) - - self.WriteLn('\n# Include paths placed before CFLAGS/CPPFLAGS') - includes = list(config.get('include_dirs', [])) - includes.extend(extracted_includes) - includes = map(Sourceify, map(self.LocalPathify, includes)) - includes = self.NormalizeIncludePaths(includes) - self.WriteList(includes, 'LOCAL_C_INCLUDES_%s' % configname) - - self.WriteLn('\n# Flags passed to only C++ (and not C) files.') - self.WriteList(config.get('cflags_cc'), 'LOCAL_CPPFLAGS_%s' % configname) - - self.WriteLn('\nLOCAL_CFLAGS := $(MY_CFLAGS_$(GYP_CONFIGURATION)) ' - '$(MY_DEFS_$(GYP_CONFIGURATION))') - # Undefine ANDROID for host modules - # TODO: the source code should not use macro ANDROID to tell if it's host - # or target module. - if self.toolset == 'host': - self.WriteLn('# Undefine ANDROID for host modules') - self.WriteLn('LOCAL_CFLAGS += -UANDROID') - self.WriteLn('LOCAL_C_INCLUDES := $(GYP_COPIED_SOURCE_ORIGIN_DIRS) ' - '$(LOCAL_C_INCLUDES_$(GYP_CONFIGURATION))') - self.WriteLn('LOCAL_CPPFLAGS := $(LOCAL_CPPFLAGS_$(GYP_CONFIGURATION))') - - - def WriteSources(self, spec, configs, extra_sources): - """Write Makefile code for any 'sources' from the gyp input. - These are source files necessary to build the current target. - We need to handle shared_intermediate directory source files as - a special case by copying them to the intermediate directory and - treating them as a genereated sources. Otherwise the Android build - rules won't pick them up. - - Args: - spec, configs: input from gyp. - extra_sources: Sources generated from Actions or Rules. - """ - sources = filter(make.Compilable, spec.get('sources', [])) - generated_not_sources = [x for x in extra_sources if not make.Compilable(x)] - extra_sources = filter(make.Compilable, extra_sources) - - # Determine and output the C++ extension used by these sources. - # We simply find the first C++ file and use that extension. - all_sources = sources + extra_sources - local_cpp_extension = '.cpp' - for source in all_sources: - (root, ext) = os.path.splitext(source) - if IsCPPExtension(ext): - local_cpp_extension = ext - break - if local_cpp_extension != '.cpp': - self.WriteLn('LOCAL_CPP_EXTENSION := %s' % local_cpp_extension) - - # We need to move any non-generated sources that are coming from the - # shared intermediate directory out of LOCAL_SRC_FILES and put them - # into LOCAL_GENERATED_SOURCES. We also need to move over any C++ files - # that don't match our local_cpp_extension, since Android will only - # generate Makefile rules for a single LOCAL_CPP_EXTENSION. - local_files = [] - for source in sources: - (root, ext) = os.path.splitext(source) - if '$(gyp_shared_intermediate_dir)' in source: - extra_sources.append(source) - elif '$(gyp_intermediate_dir)' in source: - extra_sources.append(source) - elif IsCPPExtension(ext) and ext != local_cpp_extension: - extra_sources.append(source) - else: - local_files.append(os.path.normpath(os.path.join(self.path, source))) - - # For any generated source, if it is coming from the shared intermediate - # directory then we add a Make rule to copy them to the local intermediate - # directory first. This is because the Android LOCAL_GENERATED_SOURCES - # must be in the local module intermediate directory for the compile rules - # to work properly. If the file has the wrong C++ extension, then we add - # a rule to copy that to intermediates and use the new version. - final_generated_sources = [] - # If a source file gets copied, we still need to add the orginal source - # directory as header search path, for GCC searches headers in the - # directory that contains the source file by default. - origin_src_dirs = [] - for source in extra_sources: - local_file = source - if not '$(gyp_intermediate_dir)/' in local_file: - basename = os.path.basename(local_file) - local_file = '$(gyp_intermediate_dir)/' + basename - (root, ext) = os.path.splitext(local_file) - if IsCPPExtension(ext) and ext != local_cpp_extension: - local_file = root + local_cpp_extension - if local_file != source: - self.WriteLn('%s: %s' % (local_file, self.LocalPathify(source))) - self.WriteLn('\tmkdir -p $(@D); cp $< $@') - origin_src_dirs.append(os.path.dirname(source)) - final_generated_sources.append(local_file) - - # We add back in all of the non-compilable stuff to make sure that the - # make rules have dependencies on them. - final_generated_sources.extend(generated_not_sources) - self.WriteList(final_generated_sources, 'LOCAL_GENERATED_SOURCES') - - origin_src_dirs = gyp.common.uniquer(origin_src_dirs) - origin_src_dirs = map(Sourceify, map(self.LocalPathify, origin_src_dirs)) - self.WriteList(origin_src_dirs, 'GYP_COPIED_SOURCE_ORIGIN_DIRS') - - self.WriteList(local_files, 'LOCAL_SRC_FILES') - - # Write out the flags used to compile the source; this must be done last - # so that GYP_COPIED_SOURCE_ORIGIN_DIRS can be used as an include path. - self.WriteSourceFlags(spec, configs) - - - def ComputeAndroidModule(self, spec): - """Return the Android module name used for a gyp spec. - - We use the complete qualified target name to avoid collisions between - duplicate targets in different directories. We also add a suffix to - distinguish gyp-generated module names. - """ - - if int(spec.get('android_unmangled_name', 0)): - assert self.type != 'shared_library' or self.target.startswith('lib') - return self.target - - if self.type == 'shared_library': - # For reasons of convention, the Android build system requires that all - # shared library modules are named 'libfoo' when generating -l flags. - prefix = 'lib_' - else: - prefix = '' - - if spec['toolset'] == 'host': - suffix = '_host_gyp' - else: - suffix = '_gyp' - - if self.path: - name = '%s%s_%s%s' % (prefix, self.path, self.target, suffix) - else: - name = '%s%s%s' % (prefix, self.target, suffix) - - return make.StringToMakefileVariable(name) - - - def ComputeOutputParts(self, spec): - """Return the 'output basename' of a gyp spec, split into filename + ext. - - Android libraries must be named the same thing as their module name, - otherwise the linker can't find them, so product_name and so on must be - ignored if we are building a library, and the "lib" prepending is - not done for Android. - """ - assert self.type != 'loadable_module' # TODO: not supported? - - target = spec['target_name'] - target_prefix = '' - target_ext = '' - if self.type == 'static_library': - target = self.ComputeAndroidModule(spec) - target_ext = '.a' - elif self.type == 'shared_library': - target = self.ComputeAndroidModule(spec) - target_ext = '.so' - elif self.type == 'none': - target_ext = '.stamp' - elif self.type != 'executable': - print ("ERROR: What output file should be generated?", - "type", self.type, "target", target) - - if self.type != 'static_library' and self.type != 'shared_library': - target_prefix = spec.get('product_prefix', target_prefix) - target = spec.get('product_name', target) - product_ext = spec.get('product_extension') - if product_ext: - target_ext = '.' + product_ext - - target_stem = target_prefix + target - return (target_stem, target_ext) - - - def ComputeOutputBasename(self, spec): - """Return the 'output basename' of a gyp spec. - - E.g., the loadable module 'foobar' in directory 'baz' will produce - 'libfoobar.so' - """ - return ''.join(self.ComputeOutputParts(spec)) - - - def ComputeOutput(self, spec): - """Return the 'output' (full output path) of a gyp spec. - - E.g., the loadable module 'foobar' in directory 'baz' will produce - '$(obj)/baz/libfoobar.so' - """ - if self.type == 'executable' and self.toolset == 'host': - # We install host executables into shared_intermediate_dir so they can be - # run by gyp rules that refer to PRODUCT_DIR. - path = '$(gyp_shared_intermediate_dir)' - elif self.type == 'shared_library': - if self.toolset == 'host': - path = '$(HOST_OUT_INTERMEDIATE_LIBRARIES)' - else: - path = '$(TARGET_OUT_INTERMEDIATE_LIBRARIES)' - else: - # Other targets just get built into their intermediate dir. - if self.toolset == 'host': - path = '$(call intermediates-dir-for,%s,%s,true)' % (self.android_class, - self.android_module) - else: - path = '$(call intermediates-dir-for,%s,%s)' % (self.android_class, - self.android_module) - - assert spec.get('product_dir') is None # TODO: not supported? - return os.path.join(path, self.ComputeOutputBasename(spec)) - - def NormalizeIncludePaths(self, include_paths): - """ Normalize include_paths. - Convert absolute paths to relative to the Android top directory; - filter out include paths that are already brought in by the Android build - system. - - Args: - include_paths: A list of unprocessed include paths. - Returns: - A list of normalized include paths. - """ - normalized = [] - for path in include_paths: - if path[0] == '/': - path = gyp.common.RelativePath(path, self.android_top_dir) - - # Filter out the Android standard search path. - if path not in android_standard_include_paths: - normalized.append(path) - return normalized - - def ExtractIncludesFromCFlags(self, cflags): - """Extract includes "-I..." out from cflags - - Args: - cflags: A list of compiler flags, which may be mixed with "-I.." - Returns: - A tuple of lists: (clean_clfags, include_paths). "-I.." is trimmed. - """ - clean_cflags = [] - include_paths = [] - for flag in cflags: - if flag.startswith('-I'): - include_paths.append(flag[2:]) - else: - clean_cflags.append(flag) - - return (clean_cflags, include_paths) - - def ComputeAndroidLibraryModuleNames(self, libraries): - """Compute the Android module names from libraries, ie spec.get('libraries') - - Args: - libraries: the value of spec.get('libraries') - Returns: - A tuple (static_lib_modules, dynamic_lib_modules) - """ - static_lib_modules = [] - dynamic_lib_modules = [] - for libs in libraries: - # Libs can have multiple words. - for lib in libs.split(): - # Filter the system libraries, which are added by default by the Android - # build system. - if (lib == '-lc' or lib == '-lstdc++' or lib == '-lm' or - lib.endswith('libgcc.a')): - continue - match = re.search(r'([^/]+)\.a$', lib) - if match: - static_lib_modules.append(match.group(1)) - continue - match = re.search(r'([^/]+)\.so$', lib) - if match: - dynamic_lib_modules.append(match.group(1)) - continue - # "-lstlport" -> libstlport - if lib.startswith('-l'): - if lib.endswith('_static'): - static_lib_modules.append('lib' + lib[2:]) - else: - dynamic_lib_modules.append('lib' + lib[2:]) - return (static_lib_modules, dynamic_lib_modules) - - - def ComputeDeps(self, spec): - """Compute the dependencies of a gyp spec. - - Returns a tuple (deps, link_deps), where each is a list of - filenames that will need to be put in front of make for either - building (deps) or linking (link_deps). - """ - deps = [] - link_deps = [] - if 'dependencies' in spec: - deps.extend([target_outputs[dep] for dep in spec['dependencies'] - if target_outputs[dep]]) - for dep in spec['dependencies']: - if dep in target_link_deps: - link_deps.append(target_link_deps[dep]) - deps.extend(link_deps) - return (gyp.common.uniquer(deps), gyp.common.uniquer(link_deps)) - - - def WriteTargetFlags(self, spec, configs, link_deps): - """Write Makefile code to specify the link flags and library dependencies. - - spec, configs: input from gyp. - link_deps: link dependency list; see ComputeDeps() - """ - for configname, config in sorted(configs.iteritems()): - ldflags = list(config.get('ldflags', [])) - self.WriteLn('') - self.WriteList(ldflags, 'LOCAL_LDFLAGS_%s' % configname) - self.WriteLn('\nLOCAL_LDFLAGS := $(LOCAL_LDFLAGS_$(GYP_CONFIGURATION))') - - # Libraries (i.e. -lfoo) - libraries = gyp.common.uniquer(spec.get('libraries', [])) - static_libs, dynamic_libs = self.ComputeAndroidLibraryModuleNames( - libraries) - - # Link dependencies (i.e. libfoo.a, libfoo.so) - static_link_deps = [x[1] for x in link_deps if x[0] == 'static'] - shared_link_deps = [x[1] for x in link_deps if x[0] == 'shared'] - self.WriteLn('') - self.WriteList(static_libs + static_link_deps, - 'LOCAL_STATIC_LIBRARIES') - self.WriteLn('# Enable grouping to fix circular references') - self.WriteLn('LOCAL_GROUP_STATIC_LIBRARIES := true') - self.WriteLn('') - self.WriteList(dynamic_libs + shared_link_deps, - 'LOCAL_SHARED_LIBRARIES') - - - def WriteTarget(self, spec, configs, deps, link_deps, part_of_all): - """Write Makefile code to produce the final target of the gyp spec. - - spec, configs: input from gyp. - deps, link_deps: dependency lists; see ComputeDeps() - part_of_all: flag indicating this target is part of 'all' - """ - self.WriteLn('### Rules for final target.') - - if self.type != 'none': - self.WriteTargetFlags(spec, configs, link_deps) - - # Add to the set of targets which represent the gyp 'all' target. We use the - # name 'gyp_all_modules' as the Android build system doesn't allow the use - # of the Make target 'all' and because 'all_modules' is the equivalent of - # the Make target 'all' on Android. - if part_of_all: - self.WriteLn('# Add target alias to "gyp_all_modules" target.') - self.WriteLn('.PHONY: gyp_all_modules') - self.WriteLn('gyp_all_modules: %s' % self.android_module) - self.WriteLn('') - - # Add an alias from the gyp target name to the Android module name. This - # simplifies manual builds of the target, and is required by the test - # framework. - if self.target != self.android_module: - self.WriteLn('# Alias gyp target name.') - self.WriteLn('.PHONY: %s' % self.target) - self.WriteLn('%s: %s' % (self.target, self.android_module)) - self.WriteLn('') - - # Add the command to trigger build of the target type depending - # on the toolset. Ex: BUILD_STATIC_LIBRARY vs. BUILD_HOST_STATIC_LIBRARY - # NOTE: This has to come last! - modifier = '' - if self.toolset == 'host': - modifier = 'HOST_' - if self.type == 'static_library': - self.WriteLn('include $(BUILD_%sSTATIC_LIBRARY)' % modifier) - elif self.type == 'shared_library': - self.WriteLn('LOCAL_PRELINK_MODULE := false') - self.WriteLn('include $(BUILD_%sSHARED_LIBRARY)' % modifier) - elif self.type == 'executable': - if self.toolset == 'host': - self.WriteLn('LOCAL_MODULE_PATH := $(gyp_shared_intermediate_dir)') - else: - # Don't install target executables for now, as it results in them being - # included in ROM. This can be revisited if there's a reason to install - # them later. - self.WriteLn('LOCAL_UNINSTALLABLE_MODULE := true') - self.WriteLn('include $(BUILD_%sEXECUTABLE)' % modifier) - else: - self.WriteLn('LOCAL_MODULE_PATH := $(PRODUCT_OUT)/gyp_stamp') - self.WriteLn('LOCAL_UNINSTALLABLE_MODULE := true') - self.WriteLn() - self.WriteLn('include $(BUILD_SYSTEM)/base_rules.mk') - self.WriteLn() - self.WriteLn('$(LOCAL_BUILT_MODULE): $(LOCAL_ADDITIONAL_DEPENDENCIES)') - self.WriteLn('\t$(hide) echo "Gyp timestamp: $@"') - self.WriteLn('\t$(hide) mkdir -p $(dir $@)') - self.WriteLn('\t$(hide) touch $@') - - - def WriteList(self, value_list, variable=None, prefix='', - quoter=make.QuoteIfNecessary, local_pathify=False): - """Write a variable definition that is a list of values. - - E.g. WriteList(['a','b'], 'foo', prefix='blah') writes out - foo = blaha blahb - but in a pretty-printed style. - """ - values = '' - if value_list: - value_list = [quoter(prefix + l) for l in value_list] - if local_pathify: - value_list = [self.LocalPathify(l) for l in value_list] - values = ' \\\n\t' + ' \\\n\t'.join(value_list) - self.fp.write('%s :=%s\n\n' % (variable, values)) - - - def WriteLn(self, text=''): - self.fp.write(text + '\n') - - - def LocalPathify(self, path): - """Convert a subdirectory-relative path into a normalized path which starts - with the make variable $(LOCAL_PATH) (i.e. the top of the project tree). - Absolute paths, or paths that contain variables, are just normalized.""" - if '$(' in path or os.path.isabs(path): - # path is not a file in the project tree in this case, but calling - # normpath is still important for trimming trailing slashes. - return os.path.normpath(path) - local_path = os.path.join('$(LOCAL_PATH)', self.path, path) - local_path = os.path.normpath(local_path) - # Check that normalizing the path didn't ../ itself out of $(LOCAL_PATH) - # - i.e. that the resulting path is still inside the project tree. The - # path may legitimately have ended up containing just $(LOCAL_PATH), though, - # so we don't look for a slash. - assert local_path.startswith('$(LOCAL_PATH)'), ( - 'Path %s attempts to escape from gyp path %s !)' % (path, self.path)) - return local_path - - - def ExpandInputRoot(self, template, expansion, dirname): - if '%(INPUT_ROOT)s' not in template and '%(INPUT_DIRNAME)s' not in template: - return template - path = template % { - 'INPUT_ROOT': expansion, - 'INPUT_DIRNAME': dirname, - } - return path - - -def PerformBuild(data, configurations, params): - # The android backend only supports the default configuration. - options = params['options'] - makefile = os.path.abspath(os.path.join(options.toplevel_dir, - 'GypAndroid.mk')) - env = dict(os.environ) - env['ONE_SHOT_MAKEFILE'] = makefile - arguments = ['make', '-C', os.environ['ANDROID_BUILD_TOP'], 'gyp_all_modules'] - print 'Building: %s' % arguments - subprocess.check_call(arguments, env=env) - - -def GenerateOutput(target_list, target_dicts, data, params): - options = params['options'] - generator_flags = params.get('generator_flags', {}) - builddir_name = generator_flags.get('output_dir', 'out') - limit_to_target_all = generator_flags.get('limit_to_target_all', False) - android_top_dir = os.environ.get('ANDROID_BUILD_TOP') - assert android_top_dir, '$ANDROID_BUILD_TOP not set; you need to run lunch.' - - def CalculateMakefilePath(build_file, base_name): - """Determine where to write a Makefile for a given gyp file.""" - # Paths in gyp files are relative to the .gyp file, but we want - # paths relative to the source root for the master makefile. Grab - # the path of the .gyp file as the base to relativize against. - # E.g. "foo/bar" when we're constructing targets for "foo/bar/baz.gyp". - base_path = gyp.common.RelativePath(os.path.dirname(build_file), - options.depth) - # We write the file in the base_path directory. - output_file = os.path.join(options.depth, base_path, base_name) - assert not options.generator_output, ( - 'The Android backend does not support options.generator_output.') - base_path = gyp.common.RelativePath(os.path.dirname(build_file), - options.toplevel_dir) - return base_path, output_file - - # TODO: search for the first non-'Default' target. This can go - # away when we add verification that all targets have the - # necessary configurations. - default_configuration = None - toolsets = set([target_dicts[target]['toolset'] for target in target_list]) - for target in target_list: - spec = target_dicts[target] - if spec['default_configuration'] != 'Default': - default_configuration = spec['default_configuration'] - break - if not default_configuration: - default_configuration = 'Default' - - srcdir = '.' - makefile_name = 'GypAndroid' + options.suffix + '.mk' - makefile_path = os.path.join(options.toplevel_dir, makefile_name) - assert not options.generator_output, ( - 'The Android backend does not support options.generator_output.') - gyp.common.EnsureDirExists(makefile_path) - root_makefile = open(makefile_path, 'w') - - root_makefile.write(header) - - # We set LOCAL_PATH just once, here, to the top of the project tree. This - # allows all the other paths we use to be relative to the Android.mk file, - # as the Android build system expects. - root_makefile.write('\nLOCAL_PATH := $(call my-dir)\n') - - # Find the list of targets that derive from the gyp file(s) being built. - needed_targets = set() - for build_file in params['build_files']: - for target in gyp.common.AllTargets(target_list, target_dicts, build_file): - needed_targets.add(target) - - build_files = set() - include_list = set() - android_modules = {} - for qualified_target in target_list: - build_file, target, toolset = gyp.common.ParseQualifiedTarget( - qualified_target) - relative_build_file = gyp.common.RelativePath(build_file, - options.toplevel_dir) - build_files.add(relative_build_file) - included_files = data[build_file]['included_files'] - for included_file in included_files: - # The included_files entries are relative to the dir of the build file - # that included them, so we have to undo that and then make them relative - # to the root dir. - relative_include_file = gyp.common.RelativePath( - gyp.common.UnrelativePath(included_file, build_file), - options.toplevel_dir) - abs_include_file = os.path.abspath(relative_include_file) - # If the include file is from the ~/.gyp dir, we should use absolute path - # so that relocating the src dir doesn't break the path. - if (params['home_dot_gyp'] and - abs_include_file.startswith(params['home_dot_gyp'])): - build_files.add(abs_include_file) - else: - build_files.add(relative_include_file) - - base_path, output_file = CalculateMakefilePath(build_file, - target + '.' + toolset + options.suffix + '.mk') - - spec = target_dicts[qualified_target] - configs = spec['configurations'] - - part_of_all = (qualified_target in needed_targets and - not int(spec.get('suppress_wildcard', False))) - if limit_to_target_all and not part_of_all: - continue - - relative_target = gyp.common.QualifiedTarget(relative_build_file, target, - toolset) - writer = AndroidMkWriter(android_top_dir) - android_module = writer.Write(qualified_target, relative_target, base_path, - output_file, spec, configs, - part_of_all=part_of_all) - if android_module in android_modules: - print ('ERROR: Android module names must be unique. The following ' - 'targets both generate Android module name %s.\n %s\n %s' % - (android_module, android_modules[android_module], - qualified_target)) - return - android_modules[android_module] = qualified_target - - # Our root_makefile lives at the source root. Compute the relative path - # from there to the output_file for including. - mkfile_rel_path = gyp.common.RelativePath(output_file, - os.path.dirname(makefile_path)) - include_list.add(mkfile_rel_path) - - root_makefile.write('GYP_CONFIGURATION ?= %s\n' % default_configuration) - - # Write out the sorted list of includes. - root_makefile.write('\n') - for include_file in sorted(include_list): - root_makefile.write('include $(LOCAL_PATH)/' + include_file + '\n') - root_makefile.write('\n') - - root_makefile.write(SHARED_FOOTER) - - root_makefile.close() diff --git a/tools/gyp/pylib/gyp/generator/cmake.py b/tools/gyp/pylib/gyp/generator/cmake.py index 10d015ee8315..eece6ea98ddd 100644 --- a/tools/gyp/pylib/gyp/generator/cmake.py +++ b/tools/gyp/pylib/gyp/generator/cmake.py @@ -150,20 +150,17 @@ def SetFileProperty(output, source_name, property_name, values, sep): output.write('")\n') -def SetFilesProperty(output, source_names, property_name, values, sep): +def SetFilesProperty(output, variable, property_name, values, sep): """Given a set of source files, sets the given property on them.""" - output.write('set_source_files_properties(\n') - for source_name in source_names: - output.write(' ') - output.write(source_name) - output.write('\n') - output.write(' PROPERTIES\n ') + output.write('set_source_files_properties(') + WriteVariable(output, variable) + output.write(' PROPERTIES ') output.write(property_name) output.write(' "') for value in values: output.write(CMakeStringEscape(value)) output.write(sep) - output.write('"\n)\n') + output.write('")\n') def SetTargetProperty(output, target_name, property_name, values, sep=''): @@ -216,7 +213,7 @@ def WriteVariable(output, variable_name, prepend=None): output.write('}') -class CMakeTargetType: +class CMakeTargetType(object): def __init__(self, command, modifier, property_modifier): self.command = command self.modifier = modifier @@ -236,11 +233,11 @@ def StringToCMakeTargetName(a): """Converts the given string 'a' to a valid CMake target name. All invalid characters are replaced by '_'. - Invalid for cmake: ' ', '/', '(', ')' + Invalid for cmake: ' ', '/', '(', ')', '"' Invalid for make: ':' Invalid for unknown reasons but cause failures: '.' """ - return a.translate(string.maketrans(' /():.', '______')) + return a.translate(string.maketrans(' /():."', '_______')) def WriteActions(target_name, actions, extra_sources, extra_deps, @@ -464,7 +461,7 @@ def WriteCopies(target_name, copies, extra_deps, path_to_gyp, output): extra_deps.append(copy_name) return - class Copy: + class Copy(object): def __init__(self, ext, command): self.cmake_inputs = [] self.cmake_outputs = [] @@ -488,7 +485,7 @@ def __init__(self, ext, command): copy = file_copy if os.path.basename(src) else dir_copy - copy.cmake_inputs.append(NormjoinPath(path_to_gyp, src)) + copy.cmake_inputs.append(NormjoinPathForceCMakeSource(path_to_gyp, src)) copy.cmake_outputs.append(NormjoinPathForceCMakeSource(path_to_gyp, dst)) copy.gyp_inputs.append(src) copy.gyp_outputs.append(dst) @@ -640,6 +637,12 @@ def WriteTarget(namer, qualified_target, target_dicts, build_dir, config_to_use, target_type = spec.get('type', '') target_toolset = spec.get('toolset') + cmake_target_type = cmake_target_type_from_gyp_target_type.get(target_type) + if cmake_target_type is None: + print ('Target %s has unknown target type %s, skipping.' % + ( target_name, target_type ) ) + return + SetVariable(output, 'TARGET', target_name) SetVariable(output, 'TOOLSET', target_toolset) @@ -667,27 +670,89 @@ def WriteTarget(namer, qualified_target, target_dicts, build_dir, config_to_use, srcs = spec.get('sources', []) # Gyp separates the sheep from the goats based on file extensions. - def partition(l, p): - return reduce(lambda x, e: x[not p(e)].append(e) or x, l, ([], [])) - compilable_srcs, other_srcs = partition(srcs, Compilable) + # A full separation is done here because of flag handing (see below). + s_sources = [] + c_sources = [] + cxx_sources = [] + linkable_sources = [] + other_sources = [] + for src in srcs: + _, ext = os.path.splitext(src) + src_type = COMPILABLE_EXTENSIONS.get(ext, None) + src_norm_path = NormjoinPath(path_from_cmakelists_to_gyp, src); - # CMake gets upset when executable targets provide no sources. - if target_type == 'executable' and not compilable_srcs and not extra_sources: - print ('Executable %s has no complilable sources, treating as "none".' % - target_name ) - target_type = 'none' + if src_type == 's': + s_sources.append(src_norm_path) + elif src_type == 'cc': + c_sources.append(src_norm_path) + elif src_type == 'cxx': + cxx_sources.append(src_norm_path) + elif Linkable(ext): + linkable_sources.append(src_norm_path) + else: + other_sources.append(src_norm_path) - cmake_target_type = cmake_target_type_from_gyp_target_type.get(target_type) - if cmake_target_type is None: - print ('Target %s has unknown target type %s, skipping.' % - ( target_name, target_type ) ) - return + for extra_source in extra_sources: + src, real_source = extra_source + _, ext = os.path.splitext(real_source) + src_type = COMPILABLE_EXTENSIONS.get(ext, None) + + if src_type == 's': + s_sources.append(src) + elif src_type == 'cc': + c_sources.append(src) + elif src_type == 'cxx': + cxx_sources.append(src) + elif Linkable(ext): + linkable_sources.append(src) + else: + other_sources.append(src) + + s_sources_name = None + if s_sources: + s_sources_name = cmake_target_name + '__asm_srcs' + SetVariableList(output, s_sources_name, s_sources) + + c_sources_name = None + if c_sources: + c_sources_name = cmake_target_name + '__c_srcs' + SetVariableList(output, c_sources_name, c_sources) + + cxx_sources_name = None + if cxx_sources: + cxx_sources_name = cmake_target_name + '__cxx_srcs' + SetVariableList(output, cxx_sources_name, cxx_sources) + + linkable_sources_name = None + if linkable_sources: + linkable_sources_name = cmake_target_name + '__linkable_srcs' + SetVariableList(output, linkable_sources_name, linkable_sources) + + other_sources_name = None + if other_sources: + other_sources_name = cmake_target_name + '__other_srcs' + SetVariableList(output, other_sources_name, other_sources) + + # CMake gets upset when executable targets provide no sources. + # http://www.cmake.org/pipermail/cmake/2010-July/038461.html + dummy_sources_name = None + has_sources = (s_sources_name or + c_sources_name or + cxx_sources_name or + linkable_sources_name or + other_sources_name) + if target_type == 'executable' and not has_sources: + dummy_sources_name = cmake_target_name + '__dummy_srcs' + SetVariable(output, dummy_sources_name, + "${obj}.${TOOLSET}/${TARGET}/genc/dummy.c") + output.write('if(NOT EXISTS "') + WriteVariable(output, dummy_sources_name) + output.write('")\n') + output.write(' file(WRITE "') + WriteVariable(output, dummy_sources_name) + output.write('" "")\n') + output.write("endif()\n") - other_srcs_name = None - if other_srcs: - other_srcs_name = cmake_target_name + '__other_srcs' - SetVariableList(output, other_srcs_name, - [NormjoinPath(path_from_cmakelists_to_gyp, src) for src in other_srcs]) # CMake is opposed to setting linker directories and considers the practice # of setting linker directories dangerous. Instead, it favors the use of @@ -713,37 +778,54 @@ def partition(l, p): output.write(' ') output.write(cmake_target_type.modifier) - if other_srcs_name: - WriteVariable(output, other_srcs_name, ' ') - - output.write('\n') - - for src in compilable_srcs: - output.write(' ') - output.write(NormjoinPath(path_from_cmakelists_to_gyp, src)) - output.write('\n') - for extra_source in extra_sources: - output.write(' ') - src, _ = extra_source - output.write(NormjoinPath(path_from_cmakelists_to_gyp, src)) - output.write('\n') + if s_sources_name: + WriteVariable(output, s_sources_name, ' ') + if c_sources_name: + WriteVariable(output, c_sources_name, ' ') + if cxx_sources_name: + WriteVariable(output, cxx_sources_name, ' ') + if linkable_sources_name: + WriteVariable(output, linkable_sources_name, ' ') + if other_sources_name: + WriteVariable(output, other_sources_name, ' ') + if dummy_sources_name: + WriteVariable(output, dummy_sources_name, ' ') output.write(')\n') + # Let CMake know if the 'all' target should depend on this target. + exclude_from_all = ('TRUE' if qualified_target not in all_qualified_targets + else 'FALSE') + SetTargetProperty(output, cmake_target_name, + 'EXCLUDE_FROM_ALL', exclude_from_all) + for extra_target_name in extra_deps: + SetTargetProperty(output, extra_target_name, + 'EXCLUDE_FROM_ALL', exclude_from_all) + # Output name and location. if target_type != 'none': + # Link as 'C' if there are no other files + if not c_sources and not cxx_sources: + SetTargetProperty(output, cmake_target_name, 'LINKER_LANGUAGE', ['C']) + # Mark uncompiled sources as uncompiled. - if other_srcs_name: + if other_sources_name: output.write('set_source_files_properties(') - WriteVariable(output, other_srcs_name, '') + WriteVariable(output, other_sources_name, '') output.write(' PROPERTIES HEADER_FILE_ONLY "TRUE")\n') + # Mark object sources as linkable. + if linkable_sources_name: + output.write('set_source_files_properties(') + WriteVariable(output, other_sources_name, '') + output.write(' PROPERTIES EXTERNAL_OBJECT "TRUE")\n') + # Output directory target_output_directory = spec.get('product_dir') if target_output_directory is None: if target_type in ('executable', 'loadable_module'): target_output_directory = generator_default_variables['PRODUCT_DIR'] - elif target_type in ('shared_library'): + elif target_type == 'shared_library': target_output_directory = '${builddir}/lib.${TOOLSET}' elif spec.get('standalone_static_library', False): target_output_directory = generator_default_variables['PRODUCT_DIR'] @@ -804,122 +886,84 @@ def partition(l, p): cmake_target_output_basename) SetFileProperty(output, cmake_target_output, 'GENERATED', ['TRUE'], '') - # Let CMake know if the 'all' target should depend on this target. - exclude_from_all = ('TRUE' if qualified_target not in all_qualified_targets - else 'FALSE') - SetTargetProperty(output, cmake_target_name, - 'EXCLUDE_FROM_ALL', exclude_from_all) - for extra_target_name in extra_deps: - SetTargetProperty(output, extra_target_name, - 'EXCLUDE_FROM_ALL', exclude_from_all) - - # Includes - includes = config.get('include_dirs') - if includes: - # This (target include directories) is what requires CMake 2.8.8 - includes_name = cmake_target_name + '__include_dirs' - SetVariableList(output, includes_name, - [NormjoinPathForceCMakeSource(path_from_cmakelists_to_gyp, include) - for include in includes]) - output.write('set_property(TARGET ') - output.write(cmake_target_name) - output.write(' APPEND PROPERTY INCLUDE_DIRECTORIES ') - WriteVariable(output, includes_name, '') - output.write(')\n') - - # Defines - defines = config.get('defines') - if defines is not None: - SetTargetProperty(output, - cmake_target_name, - 'COMPILE_DEFINITIONS', - defines, - ';') - - # Compile Flags - http://www.cmake.org/Bug/view.php?id=6493 - # CMake currently does not have target C and CXX flags. - # So, instead of doing... - - # cflags_c = config.get('cflags_c') - # if cflags_c is not None: - # SetTargetProperty(output, cmake_target_name, - # 'C_COMPILE_FLAGS', cflags_c, ' ') - - # cflags_cc = config.get('cflags_cc') - # if cflags_cc is not None: - # SetTargetProperty(output, cmake_target_name, - # 'CXX_COMPILE_FLAGS', cflags_cc, ' ') - - # Instead we must... - s_sources = [] - c_sources = [] - cxx_sources = [] - for src in srcs: - _, ext = os.path.splitext(src) - src_type = COMPILABLE_EXTENSIONS.get(ext, None) - - if src_type == 's': - s_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cc': - c_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cxx': - cxx_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - for extra_source in extra_sources: - src, real_source = extra_source - _, ext = os.path.splitext(real_source) - src_type = COMPILABLE_EXTENSIONS.get(ext, None) - - if src_type == 's': - s_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cc': - c_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - if src_type == 'cxx': - cxx_sources.append(NormjoinPath(path_from_cmakelists_to_gyp, src)) - - cflags = config.get('cflags', []) - cflags_c = config.get('cflags_c', []) - cflags_cxx = config.get('cflags_cc', []) - if c_sources and not (s_sources or cxx_sources): - flags = [] - flags.extend(cflags) - flags.extend(cflags_c) - SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - - elif cxx_sources and not (s_sources or c_sources): - flags = [] - flags.extend(cflags) - flags.extend(cflags_cxx) - SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - - else: - if s_sources and cflags: - SetFilesProperty(output, s_sources, 'COMPILE_FLAGS', cflags, ' ') + # Includes + includes = config.get('include_dirs') + if includes: + # This (target include directories) is what requires CMake 2.8.8 + includes_name = cmake_target_name + '__include_dirs' + SetVariableList(output, includes_name, + [NormjoinPathForceCMakeSource(path_from_cmakelists_to_gyp, include) + for include in includes]) + output.write('set_property(TARGET ') + output.write(cmake_target_name) + output.write(' APPEND PROPERTY INCLUDE_DIRECTORIES ') + WriteVariable(output, includes_name, '') + output.write(')\n') - if c_sources and (cflags or cflags_c): + # Defines + defines = config.get('defines') + if defines is not None: + SetTargetProperty(output, + cmake_target_name, + 'COMPILE_DEFINITIONS', + defines, + ';') + + # Compile Flags - http://www.cmake.org/Bug/view.php?id=6493 + # CMake currently does not have target C and CXX flags. + # So, instead of doing... + + # cflags_c = config.get('cflags_c') + # if cflags_c is not None: + # SetTargetProperty(output, cmake_target_name, + # 'C_COMPILE_FLAGS', cflags_c, ' ') + + # cflags_cc = config.get('cflags_cc') + # if cflags_cc is not None: + # SetTargetProperty(output, cmake_target_name, + # 'CXX_COMPILE_FLAGS', cflags_cc, ' ') + + # Instead we must... + cflags = config.get('cflags', []) + cflags_c = config.get('cflags_c', []) + cflags_cxx = config.get('cflags_cc', []) + if (not cflags_c or not c_sources) and (not cflags_cxx or not cxx_sources): + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', cflags, ' ') + + elif c_sources and not (s_sources or cxx_sources): flags = [] flags.extend(cflags) flags.extend(cflags_c) - SetFilesProperty(output, c_sources, 'COMPILE_FLAGS', flags, ' ') + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - if cxx_sources and (cflags or cflags_cxx): + elif cxx_sources and not (s_sources or c_sources): flags = [] flags.extend(cflags) flags.extend(cflags_cxx) - SetFilesProperty(output, cxx_sources, 'COMPILE_FLAGS', flags, ' ') + SetTargetProperty(output, cmake_target_name, 'COMPILE_FLAGS', flags, ' ') - # Have assembly link as c if there are no other files - if not c_sources and not cxx_sources and s_sources: - SetTargetProperty(output, cmake_target_name, 'LINKER_LANGUAGE', ['C']) - - # Linker flags - ldflags = config.get('ldflags') - if ldflags is not None: - SetTargetProperty(output, cmake_target_name, 'LINK_FLAGS', ldflags, ' ') + else: + # TODO: This is broken, one cannot generally set properties on files, + # as other targets may require different properties on the same files. + if s_sources and cflags: + SetFilesProperty(output, s_sources_name, 'COMPILE_FLAGS', cflags, ' ') + + if c_sources and (cflags or cflags_c): + flags = [] + flags.extend(cflags) + flags.extend(cflags_c) + SetFilesProperty(output, c_sources_name, 'COMPILE_FLAGS', flags, ' ') + + if cxx_sources and (cflags or cflags_cxx): + flags = [] + flags.extend(cflags) + flags.extend(cflags_cxx) + SetFilesProperty(output, cxx_sources_name, 'COMPILE_FLAGS', flags, ' ') + + # Linker flags + ldflags = config.get('ldflags') + if ldflags is not None: + SetTargetProperty(output, cmake_target_name, 'LINK_FLAGS', ldflags, ' ') # Note on Dependencies and Libraries: # CMake wants to handle link order, resolving the link line up front. @@ -1040,19 +1084,48 @@ def GenerateOutputForConfig(target_list, target_dicts, data, output.write('cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)\n') output.write('cmake_policy(VERSION 2.8.8)\n') - _, project_target, _ = gyp.common.ParseQualifiedTarget(target_list[-1]) + gyp_file, project_target, _ = gyp.common.ParseQualifiedTarget(target_list[-1]) output.write('project(') output.write(project_target) output.write(')\n') SetVariable(output, 'configuration', config_to_use) + ar = None + cc = None + cxx = None + + make_global_settings = data[gyp_file].get('make_global_settings', []) + build_to_top = gyp.common.InvertRelativePath(build_dir, + options.toplevel_dir) + for key, value in make_global_settings: + if key == 'AR': + ar = os.path.join(build_to_top, value) + if key == 'CC': + cc = os.path.join(build_to_top, value) + if key == 'CXX': + cxx = os.path.join(build_to_top, value) + + ar = gyp.common.GetEnvironFallback(['AR_target', 'AR'], ar) + cc = gyp.common.GetEnvironFallback(['CC_target', 'CC'], cc) + cxx = gyp.common.GetEnvironFallback(['CXX_target', 'CXX'], cxx) + + if ar: + SetVariable(output, 'CMAKE_AR', ar) + if cc: + SetVariable(output, 'CMAKE_C_COMPILER', cc) + if cxx: + SetVariable(output, 'CMAKE_CXX_COMPILER', cxx) + # The following appears to be as-yet undocumented. # http://public.kitware.com/Bug/view.php?id=8392 output.write('enable_language(ASM)\n') # ASM-ATT does not support .S files. # output.write('enable_language(ASM-ATT)\n') + if cc: + SetVariable(output, 'CMAKE_ASM_COMPILER', cc) + SetVariable(output, 'builddir', '${CMAKE_BINARY_DIR}') SetVariable(output, 'obj', '${builddir}/obj') output.write('\n') @@ -1066,6 +1139,11 @@ def GenerateOutputForConfig(target_list, target_dicts, data, output.write('set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE 1)\n') output.write('\n') + # Force ninja to use rsp files. Otherwise link and ar lines can get too long, + # resulting in 'Argument list too long' errors. + output.write('set(CMAKE_NINJA_FORCE_RESPONSE_FILE 1)\n') + output.write('\n') + namer = CMakeNamer(target_list) # The list of targets upon which the 'all' target should depend. diff --git a/tools/gyp/pylib/gyp/generator/dump_dependency_json.py b/tools/gyp/pylib/gyp/generator/dump_dependency_json.py index 927ba6ebad77..160eafe2efec 100644 --- a/tools/gyp/pylib/gyp/generator/dump_dependency_json.py +++ b/tools/gyp/pylib/gyp/generator/dump_dependency_json.py @@ -14,6 +14,9 @@ generator_wants_static_library_dependencies_adjusted = False +generator_filelist_paths = { +} + generator_default_variables = { } for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR', @@ -56,6 +59,17 @@ def CalculateGeneratorInputInfo(params): global generator_wants_static_library_dependencies_adjusted generator_wants_static_library_dependencies_adjusted = True + toplevel = params['options'].toplevel_dir + generator_dir = os.path.relpath(params['options'].generator_output or '.') + # output_dir: relative path from generator_dir to the build directory. + output_dir = generator_flags.get('output_dir', 'out') + qualified_out_dir = os.path.normpath(os.path.join( + toplevel, generator_dir, output_dir, 'gypfiles')) + global generator_filelist_paths + generator_filelist_paths = { + 'toplevel': toplevel, + 'qualified_out_dir': qualified_out_dir, + } def GenerateOutput(target_list, target_dicts, data, params): # Map of target -> list of targets it depends on. @@ -74,7 +88,11 @@ def GenerateOutput(target_list, target_dicts, data, params): edges[target].append(dep) targets_to_visit.append(dep) - filename = 'dump.json' + try: + filepath = params['generator_flags']['output_dir'] + except KeyError: + filepath = '.' + filename = os.path.join(filepath, 'dump.json') f = open(filename, 'w') json.dump(edges, f) f.close() diff --git a/tools/gyp/pylib/gyp/generator/eclipse.py b/tools/gyp/pylib/gyp/generator/eclipse.py index 84380b04da24..3544347b3bac 100644 --- a/tools/gyp/pylib/gyp/generator/eclipse.py +++ b/tools/gyp/pylib/gyp/generator/eclipse.py @@ -24,6 +24,7 @@ import gyp.common import gyp.msvs_emulation import shlex +import xml.etree.cElementTree as ET generator_wants_static_library_dependencies_adjusted = False @@ -31,8 +32,8 @@ } for dirname in ['INTERMEDIATE_DIR', 'PRODUCT_DIR', 'LIB_DIR', 'SHARED_LIB_DIR']: - # Some gyp steps fail if these are empty(!). - generator_default_variables[dirname] = 'dir' + # Some gyp steps fail if these are empty(!), so we convert them to variables + generator_default_variables[dirname] = '$' + dirname for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME', 'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT', @@ -77,7 +78,8 @@ def CalculateGeneratorInputInfo(params): def GetAllIncludeDirectories(target_list, target_dicts, - shared_intermediate_dirs, config_name, params): + shared_intermediate_dirs, config_name, params, + compiler_path): """Calculate the set of include directories to be used. Returns: @@ -88,6 +90,33 @@ def GetAllIncludeDirectories(target_list, target_dicts, gyp_includes_set = set() compiler_includes_list = [] + # Find compiler's default include dirs. + if compiler_path: + command = shlex.split(compiler_path) + command.extend(['-E', '-xc++', '-v', '-']) + proc = subprocess.Popen(args=command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.communicate()[1] + # Extract the list of include dirs from the output, which has this format: + # ... + # #include "..." search starts here: + # #include <...> search starts here: + # /usr/include/c++/4.6 + # /usr/local/include + # End of search list. + # ... + in_include_list = False + for line in output.splitlines(): + if line.startswith('#include'): + in_include_list = True + continue + if line.startswith('End of search list.'): + break + if in_include_list: + include_dir = line.strip() + if include_dir not in compiler_includes_list: + compiler_includes_list.append(include_dir) + flavor = gyp.common.GetFlavor(params) if flavor == 'win': generator_flags = params.get('generator_flags', {}) @@ -106,11 +135,10 @@ def GetAllIncludeDirectories(target_list, target_dicts, else: cflags = config['cflags'] for cflag in cflags: - include_dir = '' if cflag.startswith('-I'): include_dir = cflag[2:] - if include_dir and not include_dir in compiler_includes_list: - compiler_includes_list.append(include_dir) + if include_dir not in compiler_includes_list: + compiler_includes_list.append(include_dir) # Find standard gyp include dirs. if config.has_key('include_dirs'): @@ -125,9 +153,7 @@ def GetAllIncludeDirectories(target_list, target_dicts, include_dir = base_dir + '/' + include_dir include_dir = os.path.abspath(include_dir) - if not include_dir in gyp_includes_set: - gyp_includes_set.add(include_dir) - + gyp_includes_set.add(include_dir) # Generate a list that has all the include dirs. all_includes_list = list(gyp_includes_set) @@ -140,7 +166,7 @@ def GetAllIncludeDirectories(target_list, target_dicts, return all_includes_list -def GetCompilerPath(target_list, target_dicts, data): +def GetCompilerPath(target_list, data, options): """Determine a command that can be used to invoke the compiler. Returns: @@ -148,13 +174,12 @@ def GetCompilerPath(target_list, target_dicts, data): the compiler from that. Otherwise, see if a compiler was specified via the CC_target environment variable. """ - # First, see if the compiler is configured in make's settings. build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0]) make_global_settings_dict = data[build_file].get('make_global_settings', {}) for key, value in make_global_settings_dict: if key in ['CC', 'CXX']: - return value + return os.path.join(options.toplevel_dir, value) # Check to see if the compiler was specified as an environment variable. for key in ['CC_target', 'CC', 'CXX']: @@ -165,7 +190,8 @@ def GetCompilerPath(target_list, target_dicts, data): return 'gcc' -def GetAllDefines(target_list, target_dicts, data, config_name, params): +def GetAllDefines(target_list, target_dicts, data, config_name, params, + compiler_path): """Calculate the defines for a project. Returns: @@ -202,9 +228,8 @@ def GetAllDefines(target_list, target_dicts, data, config_name, params): # Get default compiler defines (if possible). if flavor == 'win': return all_defines # Default defines already processed in the loop above. - cc_target = GetCompilerPath(target_list, target_dicts, data) - if cc_target: - command = shlex.split(cc_target) + if compiler_path: + command = shlex.split(compiler_path) command.extend(['-E', '-dM', '-']) cpp_proc = subprocess.Popen(args=command, cwd='.', stdin=subprocess.PIPE, stdout=subprocess.PIPE) @@ -270,31 +295,123 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, shared_intermediate_dirs = [os.path.join(toplevel_build, 'obj', 'gen'), os.path.join(toplevel_build, 'gen')] - out_name = os.path.join(toplevel_build, 'eclipse-cdt-settings.xml') + GenerateCdtSettingsFile(target_list, + target_dicts, + data, + params, + config_name, + os.path.join(toplevel_build, + 'eclipse-cdt-settings.xml'), + options, + shared_intermediate_dirs) + GenerateClasspathFile(target_list, + target_dicts, + options.toplevel_dir, + toplevel_build, + os.path.join(toplevel_build, + 'eclipse-classpath.xml')) + + +def GenerateCdtSettingsFile(target_list, target_dicts, data, params, + config_name, out_name, options, + shared_intermediate_dirs): gyp.common.EnsureDirExists(out_name) - out = open(out_name, 'w') + with open(out_name, 'w') as out: + out.write('\n') + out.write('\n') + + eclipse_langs = ['C++ Source File', 'C Source File', 'Assembly Source File', + 'GNU C++', 'GNU C', 'Assembly'] + compiler_path = GetCompilerPath(target_list, data, options) + include_dirs = GetAllIncludeDirectories(target_list, target_dicts, + shared_intermediate_dirs, + config_name, params, compiler_path) + WriteIncludePaths(out, eclipse_langs, include_dirs) + defines = GetAllDefines(target_list, target_dicts, data, config_name, + params, compiler_path) + WriteMacros(out, eclipse_langs, defines) + + out.write('\n') + + +def GenerateClasspathFile(target_list, target_dicts, toplevel_dir, + toplevel_build, out_name): + '''Generates a classpath file suitable for symbol navigation and code + completion of Java code (such as in Android projects) by finding all + .java and .jar files used as action inputs.''' + gyp.common.EnsureDirExists(out_name) + result = ET.Element('classpath') + + def AddElements(kind, paths): + # First, we need to normalize the paths so they are all relative to the + # toplevel dir. + rel_paths = set() + for path in paths: + if os.path.isabs(path): + rel_paths.add(os.path.relpath(path, toplevel_dir)) + else: + rel_paths.add(path) - out.write('\n') - out.write('\n') + for path in sorted(rel_paths): + entry_element = ET.SubElement(result, 'classpathentry') + entry_element.set('kind', kind) + entry_element.set('path', path) - eclipse_langs = ['C++ Source File', 'C Source File', 'Assembly Source File', - 'GNU C++', 'GNU C', 'Assembly'] - include_dirs = GetAllIncludeDirectories(target_list, target_dicts, - shared_intermediate_dirs, config_name, - params) - WriteIncludePaths(out, eclipse_langs, include_dirs) - defines = GetAllDefines(target_list, target_dicts, data, config_name, params) - WriteMacros(out, eclipse_langs, defines) + AddElements('lib', GetJavaJars(target_list, target_dicts, toplevel_dir)) + AddElements('src', GetJavaSourceDirs(target_list, target_dicts, toplevel_dir)) + # Include the standard JRE container and a dummy out folder + AddElements('con', ['org.eclipse.jdt.launching.JRE_CONTAINER']) + # Include a dummy out folder so that Eclipse doesn't use the default /bin + # folder in the root of the project. + AddElements('output', [os.path.join(toplevel_build, '.eclipse-java-build')]) - out.write('\n') - out.close() + ET.ElementTree(result).write(out_name) + + +def GetJavaJars(target_list, target_dicts, toplevel_dir): + '''Generates a sequence of all .jars used as inputs.''' + for target_name in target_list: + target = target_dicts[target_name] + for action in target.get('actions', []): + for input_ in action['inputs']: + if os.path.splitext(input_)[1] == '.jar' and not input_.startswith('$'): + if os.path.isabs(input_): + yield input_ + else: + yield os.path.join(os.path.dirname(target_name), input_) + + +def GetJavaSourceDirs(target_list, target_dicts, toplevel_dir): + '''Generates a sequence of all likely java package root directories.''' + for target_name in target_list: + target = target_dicts[target_name] + for action in target.get('actions', []): + for input_ in action['inputs']: + if (os.path.splitext(input_)[1] == '.java' and + not input_.startswith('$')): + dir_ = os.path.dirname(os.path.join(os.path.dirname(target_name), + input_)) + # If there is a parent 'src' or 'java' folder, navigate up to it - + # these are canonical package root names in Chromium. This will + # break if 'src' or 'java' exists in the package structure. This + # could be further improved by inspecting the java file for the + # package name if this proves to be too fragile in practice. + parent_search = dir_ + while os.path.basename(parent_search) not in ['src', 'java']: + parent_search, _ = os.path.split(parent_search) + if not parent_search or parent_search == toplevel_dir: + # Didn't find a known root, just return the original path + yield dir_ + break + else: + yield parent_search def GenerateOutput(target_list, target_dicts, data, params): """Generate an XML settings file that can be imported into a CDT project.""" if params['options'].generator_output: - raise NotImplementedError, "--generator_output not implemented for eclipse" + raise NotImplementedError("--generator_output not implemented for eclipse") user_config = params.get('generator_flags', {}).get('config', None) if user_config: diff --git a/tools/gyp/pylib/gyp/generator/gypd.py b/tools/gyp/pylib/gyp/generator/gypd.py index 22ef57f847d5..3efdb9966a69 100644 --- a/tools/gyp/pylib/gyp/generator/gypd.py +++ b/tools/gyp/pylib/gyp/generator/gypd.py @@ -39,9 +39,11 @@ # These variables should just be spit back out as variable references. _generator_identity_variables = [ + 'CONFIGURATION_NAME', 'EXECUTABLE_PREFIX', 'EXECUTABLE_SUFFIX', 'INTERMEDIATE_DIR', + 'LIB_DIR', 'PRODUCT_DIR', 'RULE_INPUT_ROOT', 'RULE_INPUT_DIRNAME', @@ -49,6 +51,11 @@ 'RULE_INPUT_NAME', 'RULE_INPUT_PATH', 'SHARED_INTERMEDIATE_DIR', + 'SHARED_LIB_DIR', + 'SHARED_LIB_PREFIX', + 'SHARED_LIB_SUFFIX', + 'STATIC_LIB_PREFIX', + 'STATIC_LIB_SUFFIX', ] # gypd doesn't define a default value for OS like many other generator diff --git a/tools/gyp/pylib/gyp/generator/make.py b/tools/gyp/pylib/gyp/generator/make.py index b88a433d3d92..5cf2fe76be28 100644 --- a/tools/gyp/pylib/gyp/generator/make.py +++ b/tools/gyp/pylib/gyp/generator/make.py @@ -29,6 +29,7 @@ import gyp.common import gyp.xcode_emulation from gyp.common import GetEnvironFallback +from gyp.common import GypError generator_default_variables = { 'EXECUTABLE_PREFIX': '', @@ -280,15 +281,7 @@ def CalculateGeneratorInputInfo(params): AR.target ?= $(AR) # C++ apps need to be linked with g++. -# -# Note: flock is used to seralize linking. Linking is a memory-intensive -# process so running parallel links can often lead to thrashing. To disable -# the serialization, override LINK via an envrionment variable as follows: -# -# export LINK=g++ -# -# This will allow make to invoke N linker processes as specified in -jN. -LINK ?= %(flock)s $(builddir)/linker.lock $(CXX.target) +LINK ?= $(CXX.target) # TODO(evan): move all cross-compilation logic to gyp-time so we don't need # to replicate this environment fallback in make as well. @@ -631,6 +624,38 @@ def QuoteSpaces(s, quote=r'\ '): return s.replace(' ', quote) +# TODO: Avoid code duplication with _ValidateSourcesForMSVSProject in msvs.py. +def _ValidateSourcesForOSX(spec, all_sources): + """Makes sure if duplicate basenames are not specified in the source list. + + Arguments: + spec: The target dictionary containing the properties of the target. + """ + if spec.get('type', None) != 'static_library': + return + + basenames = {} + for source in all_sources: + name, ext = os.path.splitext(source) + is_compiled_file = ext in [ + '.c', '.cc', '.cpp', '.cxx', '.m', '.mm', '.s', '.S'] + if not is_compiled_file: + continue + basename = os.path.basename(name) # Don't include extension. + basenames.setdefault(basename, []).append(source) + + error = '' + for basename, files in basenames.iteritems(): + if len(files) > 1: + error += ' %s: %s\n' % (basename, ' '.join(files)) + + if error: + print('static library %s has several files with the same basename:\n' % + spec['target_name'] + error + 'libtool on OS X will generate' + + ' warnings for them.') + raise GypError('Duplicate basenames in sources section, see list above') + + # Map from qualified target to path to output. target_outputs = {} # Map from qualified target to any linkable output. A subset @@ -640,7 +665,7 @@ def QuoteSpaces(s, quote=r'\ '): target_link_deps = {} -class MakefileWriter: +class MakefileWriter(object): """MakefileWriter packages up the writing of one target-specific foobar.mk. Its only real entry point is Write(), and is mostly used for namespacing. @@ -758,6 +783,10 @@ def Write(self, qualified_target, base_path, output_filename, spec, configs, # Sources. all_sources = spec.get('sources', []) + extra_sources if all_sources: + if self.flavor == 'mac': + # libtool on OS X generates warnings for duplicate basenames in the same + # target. + _ValidateSourcesForOSX(spec, all_sources) self.WriteSources( configs, deps, all_sources, extra_outputs, extra_link_deps, part_of_all, @@ -990,7 +1019,8 @@ def WriteRules(self, rules, extra_sources, extra_outputs, # accidentally writing duplicate dummy rules for those outputs. self.WriteLn('%s: obj := $(abs_obj)' % outputs[0]) self.WriteLn('%s: builddir := $(abs_builddir)' % outputs[0]) - self.WriteMakeRule(outputs, inputs + ['FORCE_DO_CMD'], actions) + self.WriteMakeRule(outputs, inputs, actions, + command="%s_%d" % (name, count)) # Spaces in rule filenames are not supported, but rule variables have # spaces in them (e.g. RULE_INPUT_PATH expands to '$(abspath $<)'). # The spaces within the variables are valid, so remove the variables @@ -1101,9 +1131,12 @@ def WriteMacBundleResources(self, resources, bundle_deps): for output, res in gyp.xcode_emulation.GetMacBundleResources( generator_default_variables['PRODUCT_DIR'], self.xcode_settings, map(Sourceify, map(self.Absolutify, resources))): - self.WriteDoCmd([output], [res], 'mac_tool,,,copy-bundle-resource', - part_of_all=True) - bundle_deps.append(output) + _, ext = os.path.splitext(output) + if ext != '.xcassets': + # Make does not supports '.xcassets' emulation. + self.WriteDoCmd([output], [res], 'mac_tool,,,copy-bundle-resource', + part_of_all=True) + bundle_deps.append(output) def WriteMacInfoPlist(self, bundle_deps): @@ -1656,6 +1689,7 @@ def WriteDoCmd(self, outputs, inputs, command, part_of_all, comment=None, self.WriteMakeRule(outputs, inputs, actions = ['$(call do_cmd,%s%s)' % (command, suffix)], comment = comment, + command = command, force = True) # Add our outputs to the list of targets we read depfiles from. # all_deps is only used for deps file reading, and for deps files we replace @@ -1666,7 +1700,7 @@ def WriteDoCmd(self, outputs, inputs, command, part_of_all, comment=None, def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, - order_only=False, force=False, phony=False): + order_only=False, force=False, phony=False, command=None): """Write a Makefile rule, with some extra tricks. outputs: a list of outputs for the rule (note: this is not directly @@ -1679,6 +1713,7 @@ def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, force: if true, include FORCE_DO_CMD as an order-only dep phony: if true, the rule does not actually generate the named output, the output is just a name to run the rule + command: (optional) command name to generate unambiguous labels """ outputs = map(QuoteSpaces, outputs) inputs = map(QuoteSpaces, inputs) @@ -1687,44 +1722,38 @@ def WriteMakeRule(self, outputs, inputs, actions=None, comment=None, self.WriteLn('# ' + comment) if phony: self.WriteLn('.PHONY: ' + ' '.join(outputs)) - # TODO(evanm): just make order_only a list of deps instead of these hacks. - if order_only: - order_insert = '| ' - pick_output = ' '.join(outputs) - else: - order_insert = '' - pick_output = outputs[0] - if force: - force_append = ' FORCE_DO_CMD' - else: - force_append = '' if actions: self.WriteLn("%s: TOOLSET := $(TOOLSET)" % outputs[0]) - self.WriteLn('%s: %s%s%s' % (pick_output, order_insert, ' '.join(inputs), - force_append)) + force_append = ' FORCE_DO_CMD' if force else '' + + if order_only: + # Order only rule: Just write a simple rule. + # TODO(evanm): just make order_only a list of deps instead of this hack. + self.WriteLn('%s: | %s%s' % + (' '.join(outputs), ' '.join(inputs), force_append)) + elif len(outputs) == 1: + # Regular rule, one output: Just write a simple rule. + self.WriteLn('%s: %s%s' % (outputs[0], ' '.join(inputs), force_append)) + else: + # Regular rule, more than one output: Multiple outputs are tricky in + # make. We will write three rules: + # - All outputs depend on an intermediate file. + # - Make .INTERMEDIATE depend on the intermediate. + # - The intermediate file depends on the inputs and executes the + # actual command. + # - The intermediate recipe will 'touch' the intermediate file. + # - The multi-output rule will have an do-nothing recipe. + intermediate = "%s.intermediate" % (command if command else self.target) + self.WriteLn('%s: %s' % (' '.join(outputs), intermediate)) + self.WriteLn('\t%s' % '@:'); + self.WriteLn('%s: %s' % ('.INTERMEDIATE', intermediate)) + self.WriteLn('%s: %s%s' % + (intermediate, ' '.join(inputs), force_append)) + actions.insert(0, '$(call do_cmd,touch)') + if actions: for action in actions: self.WriteLn('\t%s' % action) - if not order_only and len(outputs) > 1: - # If we have more than one output, a rule like - # foo bar: baz - # that for *each* output we must run the action, potentially - # in parallel. That is not what we're trying to write -- what - # we want is that we run the action once and it generates all - # the files. - # http://www.gnu.org/software/hello/manual/automake/Multiple-Outputs.html - # discusses this problem and has this solution: - # 1) Write the naive rule that would produce parallel runs of - # the action. - # 2) Make the outputs seralized on each other, so we won't start - # a parallel run until the first run finishes, at which point - # we'll have generated all the outputs and we're done. - self.WriteLn('%s: %s' % (' '.join(outputs[1:]), outputs[0])) - # Add a dummy command to the "extra outputs" rule, otherwise make seems to - # think these outputs haven't (couldn't have?) changed, and thus doesn't - # flag them as changed (i.e. include in '$?') when evaluating dependent - # rules, which in turn causes do_cmd() to skip running dependent commands. - self.WriteLn('%s: ;' % (' '.join(outputs[1:]))) self.WriteLn() @@ -2034,7 +2063,6 @@ def CalculateMakefilePath(build_file, base_name): build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0]) make_global_settings_array = data[build_file].get('make_global_settings', []) wrappers = {} - wrappers['LINK'] = '%s $(builddir)/linker.lock' % flock_command for key, value in make_global_settings_array: if key.endswith('_wrapper'): wrappers[key[:-len('_wrapper')]] = '$(abspath %s)' % value diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index d8e0872c347e..1338790ef68f 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import collections import copy import ntpath import os @@ -13,6 +12,7 @@ import gyp.common import gyp.easy_xml as easy_xml +import gyp.generator.ninja as ninja_generator import gyp.MSVSNew as MSVSNew import gyp.MSVSProject as MSVSProject import gyp.MSVSSettings as MSVSSettings @@ -21,6 +21,7 @@ import gyp.MSVSUtil as MSVSUtil import gyp.MSVSVersion as MSVSVersion from gyp.common import GypError +from gyp.common import OrderedSet # TODO: Remove once bots are on 2.7, http://crbug.com/241769 def _import_OrderedDict(): @@ -41,7 +42,7 @@ def _import_OrderedDict(): # if IncrediBuild is executed from inside Visual Studio. This regex # validates that the string looks like a GUID with all uppercase hex # letters. -VALID_MSVS_GUID_CHARS = re.compile('^[A-F0-9\-]+$') +VALID_MSVS_GUID_CHARS = re.compile(r'^[A-F0-9\-]+$') generator_default_variables = { @@ -81,6 +82,11 @@ def _import_OrderedDict(): 'msvs_external_builder_out_dir', 'msvs_external_builder_build_cmd', 'msvs_external_builder_clean_cmd', + 'msvs_external_builder_clcompile_cmd', + 'msvs_enable_winrt', + 'msvs_requires_importlibrary', + 'msvs_enable_winphone', + 'msvs_application_type_revision', ] @@ -97,46 +103,6 @@ def _import_OrderedDict(): cached_domain = None -# Based on http://code.activestate.com/recipes/576694/. -class OrderedSet(collections.MutableSet): - def __init__(self, iterable=None): - self.end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.map = {} # key --> [key, prev, next] - if iterable is not None: - self |= iterable - - def __len__(self): - return len(self.map) - - def discard(self, key): - if key in self.map: - key, prev, next = self.map.pop(key) - prev[2] = next - next[1] = prev - - def __contains__(self, key): - return key in self.map - - def add(self, key): - if key not in self.map: - end = self.end - curr = end[1] - curr[2] = end[1] = self.map[key] = [key, curr, end] - - def update(self, iterable): - for i in iterable: - if i not in self: - self.add(i) - - def __iter__(self): - end = self.end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - # TODO(gspencer): Switch the os.environ calls to be # win32api.GetDomainName() and win32api.GetUserName() once the # python version in depot_tools has been updated to work on Vista @@ -153,11 +119,11 @@ def _GetDomainAndUserName(): call = subprocess.Popen(['net', 'config', 'Workstation'], stdout=subprocess.PIPE) config = call.communicate()[0] - username_re = re.compile('^User name\s+(\S+)', re.MULTILINE) + username_re = re.compile(r'^User name\s+(\S+)', re.MULTILINE) username_match = username_re.search(config) if username_match: username = username_match.group(1) - domain_re = re.compile('^Logon domain\s+(\S+)', re.MULTILINE) + domain_re = re.compile(r'^Logon domain\s+(\S+)', re.MULTILINE) domain_match = domain_re.search(config) if domain_match: domain = domain_match.group(1) @@ -266,7 +232,8 @@ def _ConvertSourcesToFilterHierarchy(sources, prefix=None, excluded=None, for f in folders: contents = _ConvertSourcesToFilterHierarchy(folders[f], prefix=prefix + [f], excluded=excluded, - list_excluded=list_excluded) + list_excluded=list_excluded, + msvs_version=msvs_version) contents = MSVSProject.Filter(f, contents=contents) result.append(contents) return result @@ -322,7 +289,7 @@ def _BuildCommandLineForRuleRaw(spec, cmd, cygwin_shell, has_input_path, if [x for x in cmd if '$(InputDir)' in x]: input_dir_preamble = ( 'set INPUTDIR=$(InputDir)\n' - 'set INPUTDIR=%INPUTDIR:$(ProjectDir)=%\n' + 'if NOT DEFINED INPUTDIR set INPUTDIR=.\\\n' 'set INPUTDIR=%INPUTDIR:~0,-1%\n' ) else: @@ -851,23 +818,27 @@ def _GenerateRulesForMSVS(p, output_dir, options, spec, if rules_external: _GenerateExternalRules(rules_external, output_dir, spec, sources, options, actions_to_add) - _AdjustSourcesForRules(spec, rules, sources, excluded_sources) + _AdjustSourcesForRules(rules, sources, excluded_sources, False) -def _AdjustSourcesForRules(spec, rules, sources, excluded_sources): +def _AdjustSourcesForRules(rules, sources, excluded_sources, is_msbuild): # Add outputs generated by each rule (if applicable). for rule in rules: - # Done if not processing outputs as sources. - if int(rule.get('process_outputs_as_sources', False)): - # Add in the outputs from this rule. - trigger_files = _FindRuleTriggerFiles(rule, sources) - for trigger_file in trigger_files: + # Add in the outputs from this rule. + trigger_files = _FindRuleTriggerFiles(rule, sources) + for trigger_file in trigger_files: + # Remove trigger_file from excluded_sources to let the rule be triggered + # (e.g. rule trigger ax_enums.idl is added to excluded_sources + # because it's also in an action's inputs in the same project) + excluded_sources.discard(_FixPath(trigger_file)) + # Done if not processing outputs as sources. + if int(rule.get('process_outputs_as_sources', False)): inputs, outputs = _RuleInputsAndOutputs(rule, trigger_file) inputs = OrderedSet(_FixPaths(inputs)) outputs = OrderedSet(_FixPaths(outputs)) inputs.remove(_FixPath(trigger_file)) sources.update(inputs) - if not spec.get('msvs_external_builder'): + if not is_msbuild: excluded_sources.update(inputs) sources.update(outputs) @@ -954,6 +925,42 @@ def _GenerateProject(project, options, version, generator_flags): return _GenerateMSVSProject(project, options, version, generator_flags) +# TODO: Avoid code duplication with _ValidateSourcesForOSX in make.py. +def _ValidateSourcesForMSVSProject(spec, version): + """Makes sure if duplicate basenames are not specified in the source list. + + Arguments: + spec: The target dictionary containing the properties of the target. + version: The VisualStudioVersion object. + """ + # This validation should not be applied to MSVC2010 and later. + assert not version.UsesVcxproj() + + # TODO: Check if MSVC allows this for loadable_module targets. + if spec.get('type', None) not in ('static_library', 'shared_library'): + return + sources = spec.get('sources', []) + basenames = {} + for source in sources: + name, ext = os.path.splitext(source) + is_compiled_file = ext in [ + '.c', '.cc', '.cpp', '.cxx', '.m', '.mm', '.s', '.S'] + if not is_compiled_file: + continue + basename = os.path.basename(name) # Don't include extension. + basenames.setdefault(basename, []).append(source) + + error = '' + for basename, files in basenames.iteritems(): + if len(files) > 1: + error += ' %s: %s\n' % (basename, ' '.join(files)) + + if error: + print('static library %s has several files with the same basename:\n' % + spec['target_name'] + error + 'MSVC08 cannot handle that.') + raise GypError('Duplicate basenames in sources section, see list above') + + def _GenerateMSVSProject(project, options, version, generator_flags): """Generates a .vcproj file. It may create .rules and .user files too. @@ -979,6 +986,11 @@ def _GenerateMSVSProject(project, options, version, generator_flags): for config_name, config in spec['configurations'].iteritems(): _AddConfigurationToMSVSProject(p, spec, config_type, config_name, config) + # MSVC08 and prior version cannot handle duplicate basenames in the same + # target. + # TODO: Take excluded sources into consideration if possible. + _ValidateSourcesForMSVSProject(spec, version) + # Prepare list of sources and excluded sources. gyp_file = os.path.split(project.build_file)[1] sources, excluded_sources = _PrepareListOfSources(spec, generator_flags, @@ -1098,7 +1110,8 @@ def _AddConfigurationToMSVSProject(p, spec, config_type, config_name, config): for this configuration. """ # Get the information for this configuration - include_dirs, resource_include_dirs = _GetIncludeDirs(config) + include_dirs, midl_include_dirs, resource_include_dirs = \ + _GetIncludeDirs(config) libraries = _GetLibraries(spec) library_dirs = _GetLibraryDirs(config) out_file, vc_tool, _ = _GetOutputFilePathAndTool(spec, msbuild=False) @@ -1126,6 +1139,8 @@ def _AddConfigurationToMSVSProject(p, spec, config_type, config_name, config): # Add the information to the appropriate tool _ToolAppend(tools, 'VCCLCompilerTool', 'AdditionalIncludeDirectories', include_dirs) + _ToolAppend(tools, 'VCMIDLTool', + 'AdditionalIncludeDirectories', midl_include_dirs) _ToolAppend(tools, 'VCResourceCompilerTool', 'AdditionalIncludeDirectories', resource_include_dirs) # Add in libraries. @@ -1181,10 +1196,14 @@ def _GetIncludeDirs(config): include_dirs = ( config.get('include_dirs', []) + config.get('msvs_system_include_dirs', [])) + midl_include_dirs = ( + config.get('midl_include_dirs', []) + + config.get('msvs_system_include_dirs', [])) resource_include_dirs = config.get('resource_include_dirs', include_dirs) include_dirs = _FixPaths(include_dirs) + midl_include_dirs = _FixPaths(midl_include_dirs) resource_include_dirs = _FixPaths(resource_include_dirs) - return include_dirs, resource_include_dirs + return include_dirs, midl_include_dirs, resource_include_dirs def _GetLibraryDirs(config): @@ -1218,7 +1237,7 @@ def _GetLibraries(spec): found = OrderedSet() unique_libraries_list = [] for entry in reversed(libraries): - library = re.sub('^\-l', '', entry) + library = re.sub(r'^\-l', '', entry) if not os.path.splitext(library)[1]: library += '.lib' if library not in found: @@ -1478,8 +1497,14 @@ def _AdjustSourcesAndConvertToFilterHierarchy( # Prune filters with a single child to flatten ugly directory structures # such as ../../src/modules/module1 etc. - while len(sources) == 1 and isinstance(sources[0], MSVSProject.Filter): - sources = sources[0].contents + if version.UsesVcxproj(): + while all([isinstance(s, MSVSProject.Filter) for s in sources]) \ + and len(set([s.name for s in sources])) == 1: + assert all([len(s.contents) == 1 for s in sources]) + sources = [s.contents[0] for s in sources] + else: + while len(sources) == 1 and isinstance(sources[0], MSVSProject.Filter): + sources = sources[0].contents return sources, excluded_sources, excluded_idl @@ -1815,7 +1840,7 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version): return projects -def _InitNinjaFlavor(options, target_list, target_dicts): +def _InitNinjaFlavor(params, target_list, target_dicts): """Initialize targets for the ninja flavor. This sets up the necessary variables in the targets to generate msvs projects @@ -1823,7 +1848,7 @@ def _InitNinjaFlavor(options, target_list, target_dicts): if they have not been set. This allows individual specs to override the default values initialized here. Arguments: - options: Options provided to the generator. + params: Params provided to the generator. target_list: List of target pairs: 'base/base.gyp:base'. target_dicts: Dict of target properties keyed on target pair. """ @@ -1837,8 +1862,15 @@ def _InitNinjaFlavor(options, target_list, target_dicts): spec['msvs_external_builder'] = 'ninja' if not spec.get('msvs_external_builder_out_dir'): - spec['msvs_external_builder_out_dir'] = \ - options.depth + '/out/$(Configuration)' + gyp_file, _, _ = gyp.common.ParseQualifiedTarget(qualified_target) + gyp_dir = os.path.dirname(gyp_file) + configuration = '$(Configuration)' + if params.get('target_arch') == 'x64': + configuration += '_x64' + spec['msvs_external_builder_out_dir'] = os.path.join( + gyp.common.RelativePath(params['options'].toplevel_dir, gyp_dir), + ninja_generator.ComputeOutputDir(params), + configuration) if not spec.get('msvs_external_builder_build_cmd'): spec['msvs_external_builder_build_cmd'] = [ path_to_ninja, @@ -1851,8 +1883,7 @@ def _InitNinjaFlavor(options, target_list, target_dicts): path_to_ninja, '-C', '$(OutDir)', - '-t', - 'clean', + '-tclean', '$(ProjectName)', ] @@ -1933,7 +1964,7 @@ def GenerateOutput(target_list, target_dicts, data, params): # Optionally configure each spec to use ninja as the external builder. if params.get('flavor') == 'ninja': - _InitNinjaFlavor(options, target_list, target_dicts) + _InitNinjaFlavor(params, target_list, target_dicts) # Prepare the set of configurations. configs = set() @@ -1986,7 +2017,7 @@ def GenerateOutput(target_list, target_dicts, data, params): def _GenerateMSBuildFiltersFile(filters_path, source_files, - extension_to_rule_name): + rule_dependencies, extension_to_rule_name): """Generate the filters file. This file is used by Visual Studio to organize the presentation of source @@ -1999,8 +2030,8 @@ def _GenerateMSBuildFiltersFile(filters_path, source_files, """ filter_group = [] source_group = [] - _AppendFiltersForMSBuild('', source_files, extension_to_rule_name, - filter_group, source_group) + _AppendFiltersForMSBuild('', source_files, rule_dependencies, + extension_to_rule_name, filter_group, source_group) if filter_group: content = ['Project', {'ToolsVersion': '4.0', @@ -2015,7 +2046,7 @@ def _GenerateMSBuildFiltersFile(filters_path, source_files, os.unlink(filters_path) -def _AppendFiltersForMSBuild(parent_filter_name, sources, +def _AppendFiltersForMSBuild(parent_filter_name, sources, rule_dependencies, extension_to_rule_name, filter_group, source_group): """Creates the list of filters and sources to be added in the filter file. @@ -2041,11 +2072,12 @@ def _AppendFiltersForMSBuild(parent_filter_name, sources, ['UniqueIdentifier', MSVSNew.MakeGuid(source.name)]]) # Recurse and add its dependents. _AppendFiltersForMSBuild(filter_name, source.contents, - extension_to_rule_name, + rule_dependencies, extension_to_rule_name, filter_group, source_group) else: # It's a source. Create a source entry. - _, element = _MapFileToMsBuildSourceType(source, extension_to_rule_name) + _, element = _MapFileToMsBuildSourceType(source, rule_dependencies, + extension_to_rule_name) source_entry = [element, {'Include': source}] # Specify the filter it is part of, if any. if parent_filter_name: @@ -2053,7 +2085,8 @@ def _AppendFiltersForMSBuild(parent_filter_name, sources, source_group.append(source_entry) -def _MapFileToMsBuildSourceType(source, extension_to_rule_name): +def _MapFileToMsBuildSourceType(source, rule_dependencies, + extension_to_rule_name): """Returns the group and element type of the source file. Arguments: @@ -2076,9 +2109,15 @@ def _MapFileToMsBuildSourceType(source, extension_to_rule_name): elif ext == '.rc': group = 'resource' element = 'ResourceCompile' + elif ext == '.asm': + group = 'masm' + element = 'MASM' elif ext == '.idl': group = 'midl' element = 'Midl' + elif source in rule_dependencies: + group = 'rule_dependency' + element = 'CustomBuild' else: group = 'none' element = 'None' @@ -2088,7 +2127,8 @@ def _MapFileToMsBuildSourceType(source, extension_to_rule_name): def _GenerateRulesForMSBuild(output_dir, options, spec, sources, excluded_sources, props_files_of_rules, targets_files_of_rules, - actions_to_add, extension_to_rule_name): + actions_to_add, rule_dependencies, + extension_to_rule_name): # MSBuild rules are implemented using three files: an XML file, a .targets # file and a .props file. # See http://blogs.msdn.com/b/vcblog/archive/2010/04/21/quick-help-on-vs2010-custom-build-rule.aspx @@ -2104,6 +2144,7 @@ def _GenerateRulesForMSBuild(output_dir, options, spec, continue msbuild_rule = MSBuildRule(rule, spec) msbuild_rules.append(msbuild_rule) + rule_dependencies.update(msbuild_rule.additional_dependencies.split(';')) extension_to_rule_name[msbuild_rule.extension] = msbuild_rule.rule_name if msbuild_rules: base = spec['target_name'] + options.suffix @@ -2125,7 +2166,7 @@ def _GenerateRulesForMSBuild(output_dir, options, spec, if rules_external: _GenerateExternalRules(rules_external, output_dir, spec, sources, options, actions_to_add) - _AdjustSourcesForRules(spec, rules, sources, excluded_sources) + _AdjustSourcesForRules(rules, sources, excluded_sources, True) class MSBuildRule(object): @@ -2304,6 +2345,9 @@ def _GenerateMSBuildRuleTargetsFile(targets_path, msbuild_rules): rule_name, {'Condition': "'@(%s)' != '' and '%%(%s.ExcludedFromBuild)' != " "'true'" % (rule_name, rule_name), + 'EchoOff': 'true', + 'StandardOutputImportance': 'High', + 'StandardErrorImportance': 'High', 'CommandLineTemplate': '%%(%s.CommandLineTemplate)' % rule_name, 'AdditionalOptions': '%%(%s.AdditionalOptions)' % rule_name, 'Inputs': rule_inputs @@ -2578,14 +2622,34 @@ def _GetMSBuildProjectConfigurations(configurations): def _GetMSBuildGlobalProperties(spec, guid, gyp_file_name): namespace = os.path.splitext(gyp_file_name)[0] - return [ + properties = [ ['PropertyGroup', {'Label': 'Globals'}, - ['ProjectGuid', guid], - ['Keyword', 'Win32Proj'], - ['RootNamespace', namespace], + ['ProjectGuid', guid], + ['Keyword', 'Win32Proj'], + ['RootNamespace', namespace], + ['IgnoreWarnCompileDuplicatedFilename', 'true'], ] - ] + ] + if os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or \ + os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64': + properties[0].append(['PreferredToolArchitecture', 'x64']) + + if spec.get('msvs_enable_winrt'): + properties[0].append(['DefaultLanguage', 'en-US']) + properties[0].append(['AppContainerApplication', 'true']) + if spec.get('msvs_application_type_revision'): + app_type_revision = spec.get('msvs_application_type_revision') + properties[0].append(['ApplicationTypeRevision', app_type_revision]) + else: + properties[0].append(['ApplicationTypeRevision', '8.1']) + + if spec.get('msvs_enable_winphone'): + properties[0].append(['ApplicationType', 'Windows Phone']) + else: + properties[0].append(['ApplicationType', 'Windows Store']) + + return properties def _GetMSBuildConfigurationDetails(spec, build_file): properties = {} @@ -2596,8 +2660,9 @@ def _GetMSBuildConfigurationDetails(spec, build_file): _AddConditionalProperty(properties, condition, 'ConfigurationType', msbuild_attributes['ConfigurationType']) if character_set: - _AddConditionalProperty(properties, condition, 'CharacterSet', - character_set) + if 'msvs_enable_winrt' not in spec : + _AddConditionalProperty(properties, condition, 'CharacterSet', + character_set) return _GetMSBuildPropertyGroup(spec, 'Configuration', properties) @@ -2812,7 +2877,7 @@ def _AddConditionalProperty(properties, condition, name, value): # Regex for msvs variable references ( i.e. $(FOO) ). -MSVS_VARIABLE_REFERENCE = re.compile('\$\(([a-zA-Z_][a-zA-Z0-9_]*)\)') +MSVS_VARIABLE_REFERENCE = re.compile(r'\$\(([a-zA-Z_][a-zA-Z0-9_]*)\)') def _GetMSBuildPropertyGroup(spec, label, properties): @@ -2896,7 +2961,8 @@ def _FinalizeMSBuildSettings(spec, configuration): converted = True msvs_settings = configuration.get('msvs_settings', {}) msbuild_settings = MSVSSettings.ConvertToMSBuildSettings(msvs_settings) - include_dirs, resource_include_dirs = _GetIncludeDirs(configuration) + include_dirs, midl_include_dirs, resource_include_dirs = \ + _GetIncludeDirs(configuration) libraries = _GetLibraries(spec) library_dirs = _GetLibraryDirs(configuration) out_file, _, msbuild_tool = _GetOutputFilePathAndTool(spec, msbuild=True) @@ -2906,7 +2972,7 @@ def _FinalizeMSBuildSettings(spec, configuration): # Visual Studio 2010 has TR1 defines = [d for d in defines if d != '_HAS_TR1=0'] # Warn of ignored settings - ignored_settings = ['msvs_prebuild', 'msvs_postbuild', 'msvs_tool_files'] + ignored_settings = ['msvs_tool_files'] for ignored_setting in ignored_settings: value = configuration.get(ignored_setting) if value: @@ -2915,9 +2981,8 @@ def _FinalizeMSBuildSettings(spec, configuration): defines = [_EscapeCppDefineForMSBuild(d) for d in defines] disabled_warnings = _GetDisabledWarnings(configuration) - # TODO(jeanluc) Validate & warn that we don't translate - # prebuild = configuration.get('msvs_prebuild') - # postbuild = configuration.get('msvs_postbuild') + prebuild = configuration.get('msvs_prebuild') + postbuild = configuration.get('msvs_postbuild') def_file = _GetModuleDefinition(spec) precompiled_header = configuration.get('msvs_precompiled_header') @@ -2927,6 +2992,8 @@ def _FinalizeMSBuildSettings(spec, configuration): # if you don't have any resources. _ToolAppend(msbuild_settings, 'ClCompile', 'AdditionalIncludeDirectories', include_dirs) + _ToolAppend(msbuild_settings, 'Midl', + 'AdditionalIncludeDirectories', midl_include_dirs) _ToolAppend(msbuild_settings, 'ResourceCompile', 'AdditionalIncludeDirectories', resource_include_dirs) # Add in libraries, note that even for empty libraries, we want this @@ -2957,6 +3024,13 @@ def _FinalizeMSBuildSettings(spec, configuration): 'PrecompiledHeaderFile', precompiled_header) _ToolAppend(msbuild_settings, 'ClCompile', 'ForcedIncludeFiles', [precompiled_header]) + else: + _ToolAppend(msbuild_settings, 'ClCompile', 'PrecompiledHeader', 'NotUsing') + # Turn off WinRT compilation + _ToolAppend(msbuild_settings, 'ClCompile', 'CompileAsWinRT', 'false') + # Turn on import libraries if appropriate + if spec.get('msvs_requires_importlibrary'): + _ToolAppend(msbuild_settings, '', 'IgnoreImportLibrary', 'false') # Loadable modules don't generate import libraries; # tell dependent projects to not expect one. if spec['type'] == 'loadable_module': @@ -2965,6 +3039,10 @@ def _FinalizeMSBuildSettings(spec, configuration): if def_file: _ToolAppend(msbuild_settings, 'Link', 'ModuleDefinitionFile', def_file) configuration['finalized_msbuild_settings'] = msbuild_settings + if prebuild: + _ToolAppend(msbuild_settings, 'PreBuildEvent', 'Command', prebuild) + if postbuild: + _ToolAppend(msbuild_settings, 'PostBuildEvent', 'Command', postbuild) def _GetValueFormattedForMSBuild(tool_name, name, value): @@ -3020,15 +3098,18 @@ def _VerifySourcesExist(sources, root_dir): return missing_sources -def _GetMSBuildSources(spec, sources, exclusions, extension_to_rule_name, - actions_spec, sources_handled_by_action, list_excluded): - groups = ['none', 'midl', 'include', 'compile', 'resource', 'rule'] +def _GetMSBuildSources(spec, sources, exclusions, rule_dependencies, + extension_to_rule_name, actions_spec, + sources_handled_by_action, list_excluded): + groups = ['none', 'masm', 'midl', 'include', 'compile', 'resource', 'rule', + 'rule_dependency'] grouped_sources = {} for g in groups: grouped_sources[g] = [] _AddSources2(spec, sources, exclusions, grouped_sources, - extension_to_rule_name, sources_handled_by_action, list_excluded) + rule_dependencies, extension_to_rule_name, + sources_handled_by_action, list_excluded) sources = [] for g in groups: if grouped_sources[g]: @@ -3039,13 +3120,15 @@ def _GetMSBuildSources(spec, sources, exclusions, extension_to_rule_name, def _AddSources2(spec, sources, exclusions, grouped_sources, - extension_to_rule_name, sources_handled_by_action, + rule_dependencies, extension_to_rule_name, + sources_handled_by_action, list_excluded): extensions_excluded_from_precompile = [] for source in sources: if isinstance(source, MSVSProject.Filter): _AddSources2(spec, source.contents, exclusions, grouped_sources, - extension_to_rule_name, sources_handled_by_action, + rule_dependencies, extension_to_rule_name, + sources_handled_by_action, list_excluded) else: if not source in sources_handled_by_action: @@ -3088,7 +3171,7 @@ def _AddSources2(spec, sources, exclusions, grouped_sources, detail.append(['PrecompiledHeader', '']) detail.append(['ForcedIncludeFiles', '']) - group, element = _MapFileToMsBuildSourceType(source, + group, element = _MapFileToMsBuildSourceType(source, rule_dependencies, extension_to_rule_name) grouped_sources[group].append([element, {'Include': source}] + detail) @@ -3132,6 +3215,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): actions_to_add = {} props_files_of_rules = set() targets_files_of_rules = set() + rule_dependencies = set() extension_to_rule_name = {} list_excluded = generator_flags.get('msvs_list_excluded_files', True) @@ -3140,10 +3224,11 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): _GenerateRulesForMSBuild(project_dir, options, spec, sources, excluded_sources, props_files_of_rules, targets_files_of_rules, - actions_to_add, extension_to_rule_name) + actions_to_add, rule_dependencies, + extension_to_rule_name) else: rules = spec.get('rules', []) - _AdjustSourcesForRules(spec, rules, sources, excluded_sources) + _AdjustSourcesForRules(rules, sources, excluded_sources, True) sources, excluded_sources, excluded_idl = ( _AdjustSourcesAndConvertToFilterHierarchy(spec, options, @@ -3166,6 +3251,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): spec, actions_to_add) _GenerateMSBuildFiltersFile(project.path + '.filters', sources, + rule_dependencies, extension_to_rule_name) missing_sources = _VerifySourcesExist(sources, project_dir) @@ -3180,6 +3266,12 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): ['Import', {'Project': r'$(VCTargetsPath)\Microsoft.Cpp.props'}]] import_cpp_targets_section = [ ['Import', {'Project': r'$(VCTargetsPath)\Microsoft.Cpp.targets'}]] + import_masm_props_section = [ + ['Import', + {'Project': r'$(VCTargetsPath)\BuildCustomizations\masm.props'}]] + import_masm_targets_section = [ + ['Import', + {'Project': r'$(VCTargetsPath)\BuildCustomizations\masm.targets'}]] macro_section = [['PropertyGroup', {'Label': 'UserMacros'}]] content = [ @@ -3193,8 +3285,12 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): content += _GetMSBuildGlobalProperties(spec, project.guid, project_file_name) content += import_default_section content += _GetMSBuildConfigurationDetails(spec, project.build_file) - content += _GetMSBuildLocalProperties(project.msbuild_toolset) + if spec.get('msvs_enable_winphone'): + content += _GetMSBuildLocalProperties('v120_wp81') + else: + content += _GetMSBuildLocalProperties(project.msbuild_toolset) content += import_cpp_props_section + content += import_masm_props_section content += _GetMSBuildExtensions(props_files_of_rules) content += _GetMSBuildPropertySheets(configurations) content += macro_section @@ -3202,10 +3298,11 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): project.build_file) content += _GetMSBuildToolSettingsSections(spec, configurations) content += _GetMSBuildSources( - spec, sources, exclusions, extension_to_rule_name, actions_spec, - sources_handled_by_action, list_excluded) + spec, sources, exclusions, rule_dependencies, extension_to_rule_name, + actions_spec, sources_handled_by_action, list_excluded) content += _GetMSBuildProjectReferences(project) content += import_cpp_targets_section + content += import_masm_targets_section content += _GetMSBuildExtensionTargets(targets_files_of_rules) if spec.get('msvs_external_builder'): @@ -3222,7 +3319,9 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): def _GetMSBuildExternalBuilderTargets(spec): """Return a list of MSBuild targets for external builders. - Right now, only "Build" and "Clean" targets are generated. + The "Build" and "Clean" targets are always generated. If the spec contains + 'msvs_external_builder_clcompile_cmd', then the "ClCompile" target will also + be generated, to support building selected C/C++ files. Arguments: spec: The gyp target spec. @@ -3241,7 +3340,17 @@ def _GetMSBuildExternalBuilderTargets(spec): clean_target = ['Target', {'Name': 'Clean'}] clean_target.append(['Exec', {'Command': clean_cmd}]) - return [build_target, clean_target] + targets = [build_target, clean_target] + + if spec.get('msvs_external_builder_clcompile_cmd'): + clcompile_cmd = _BuildCommandLineForRuleRaw( + spec, spec['msvs_external_builder_clcompile_cmd'], + False, False, False, False) + clcompile_target = ['Target', {'Name': 'ClCompile'}] + clcompile_target.append(['Exec', {'Command': clcompile_cmd}]) + targets.append(clcompile_target) + + return targets def _GetMSBuildExtensions(props_files_of_rules): @@ -3295,8 +3404,8 @@ def _GenerateActionsForMSBuild(spec, actions_to_add): # get too long. See also _AddActions: cygwin's setup_env mustn't be called # for every invocation or the command that sets the PATH will grow too # long. - command = ( - '\r\nif %errorlevel% neq 0 exit /b %errorlevel%\r\n'.join(commands)) + command = '\r\n'.join([c + '\r\nif %errorlevel% neq 0 exit /b %errorlevel%' + for c in commands]) _AddMSBuildAction(spec, primary_input, inputs, diff --git a/tools/gyp/pylib/gyp/generator/ninja.py b/tools/gyp/pylib/gyp/generator/ninja.py index 746181474525..c2437822a7f8 100644 --- a/tools/gyp/pylib/gyp/generator/ninja.py +++ b/tools/gyp/pylib/gyp/generator/ninja.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import collections import copy import hashlib import json @@ -13,6 +14,7 @@ import sys import gyp import gyp.common +from gyp.common import OrderedSet import gyp.msvs_emulation import gyp.MSVSUtil as MSVSUtil import gyp.xcode_emulation @@ -60,17 +62,7 @@ generator_extra_sources_for_rules = [] generator_filelist_paths = None -# TODO: figure out how to not build extra host objects in the non-cross-compile -# case when this is enabled, and enable unconditionally. -generator_supports_multiple_toolsets = ( - os.environ.get('GYP_CROSSCOMPILE') or - os.environ.get('AR_host') or - os.environ.get('CC_host') or - os.environ.get('CXX_host') or - os.environ.get('AR_target') or - os.environ.get('CC_target') or - os.environ.get('CXX_target')) - +generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested() def StripPrefix(arg, prefix): if arg.startswith(prefix): @@ -106,7 +98,7 @@ def AddArch(output, arch): return '%s.%s%s' % (output, arch, extension) -class Target: +class Target(object): """Target represents the paths used within a single gyp target. Conceptually, building a single target A is a series of steps: @@ -210,8 +202,8 @@ def FinalOutput(self): # an output file; the result can be namespaced such that it is unique # to the input file name as well as the output target name. -class NinjaWriter: - def __init__(self, qualified_target, target_outputs, base_dir, build_dir, +class NinjaWriter(object): + def __init__(self, hash_for_rules, target_outputs, base_dir, build_dir, output_file, toplevel_build, output_file_name, flavor, toplevel_dir=None): """ @@ -221,7 +213,7 @@ def __init__(self, qualified_target, target_outputs, base_dir, build_dir, toplevel_dir: path to the toplevel directory """ - self.qualified_target = qualified_target + self.hash_for_rules = hash_for_rules self.target_outputs = target_outputs self.base_dir = base_dir self.build_dir = build_dir @@ -338,12 +330,15 @@ def GypPathToUniqueOutput(self, path, qualified=True): obj += '.' + self.toolset path_dir, path_basename = os.path.split(path) + assert not os.path.isabs(path_dir), ( + "'%s' can not be absolute path (see crbug.com/462153)." % path_dir) + if qualified: path_basename = self.name + '.' + path_basename return os.path.normpath(os.path.join(obj, self.base_dir, path_dir, path_basename)) - def WriteCollapsedDependencies(self, name, targets): + def WriteCollapsedDependencies(self, name, targets, order_only=None): """Given a list of targets, return a path for a single file representing the result of building all the targets or None. @@ -351,10 +346,11 @@ def WriteCollapsedDependencies(self, name, targets): assert targets == filter(None, targets), targets if len(targets) == 0: + assert not order_only return None - if len(targets) > 1: + if len(targets) > 1 or order_only: stamp = self.GypPathToUniqueOutput(name + '.stamp') - targets = self.ninja.build(stamp, 'stamp', targets) + targets = self.ninja.build(stamp, 'stamp', targets, order_only=order_only) self.ninja.newline() return targets[0] @@ -391,6 +387,9 @@ def WriteSpec(self, spec, config_name, generator_flags): self.ninja.variable('arch', self.win_env[arch]) self.ninja.variable('cc', '$cl_' + arch) self.ninja.variable('cxx', '$cl_' + arch) + self.ninja.variable('cc_host', '$cl_' + arch) + self.ninja.variable('cxx_host', '$cl_' + arch) + self.ninja.variable('asm', '$ml_' + arch) if self.flavor == 'mac': self.archs = self.xcode_settings.GetActiveArchs(config_name) @@ -472,6 +471,8 @@ def WriteSpec(self, spec, config_name, generator_flags): else: print "Warning: Actions/rules writing object files don't work with " \ "multiarch targets, dropping. (target %s)" % spec['target_name'] + elif self.flavor == 'mac' and len(self.archs) > 1: + link_deps = collections.defaultdict(list) if self.flavor == 'win' and self.target.type == 'static_library': @@ -523,7 +524,7 @@ def fix_path(path, rel=None): def WriteWinIdlFiles(self, spec, prebuild): """Writes rules to match MSVS's implicit idl handling.""" assert self.flavor == 'win' - if self.msvs_settings.HasExplicitIdlRules(spec): + if self.msvs_settings.HasExplicitIdlRulesOrActions(spec): return [] outputs = [] for source in filter(lambda x: x.endswith('.idl'), spec['sources']): @@ -557,9 +558,10 @@ def WriteActionsRulesCopies(self, spec, extra_sources, prebuild, stamp = self.WriteCollapsedDependencies('actions_rules_copies', outputs) if self.is_mac_bundle: - self.WriteMacBundleResources( + xcassets = self.WriteMacBundleResources( extra_mac_bundle_resources + mac_bundle_resources, mac_bundle_depends) - self.WriteMacInfoPlist(mac_bundle_depends) + partial_info_plist = self.WriteMacXCassets(xcassets, mac_bundle_depends) + self.WriteMacInfoPlist(partial_info_plist, mac_bundle_depends) return stamp @@ -580,23 +582,24 @@ def GenerateDescription(self, verb, message, fallback): def WriteActions(self, actions, extra_sources, prebuild, extra_mac_bundle_resources): # Actions cd into the base directory. - env = self.GetSortedXcodeEnv() - if self.flavor == 'win': - env = self.msvs_settings.GetVSMacroEnv( - '$!PRODUCT_DIR', config=self.config_name) + env = self.GetToolchainEnv() all_outputs = [] for action in actions: # First write out a rule for the action. - name = '%s_%s' % (action['action_name'], - hashlib.md5(self.qualified_target).hexdigest()) + name = '%s_%s' % (action['action_name'], self.hash_for_rules) description = self.GenerateDescription('ACTION', action.get('message', None), name) is_cygwin = (self.msvs_settings.IsRuleRunUnderCygwin(action) if self.flavor == 'win' else False) args = action['action'] + depfile = action.get('depfile', None) + if depfile: + depfile = self.ExpandSpecial(depfile, self.base_to_build) + pool = 'console' if int(action.get('ninja_use_console', 0)) else None rule_name, _ = self.WriteNewNinjaRule(name, args, description, - is_cygwin, env=env) + is_cygwin, env, pool, + depfile=depfile) inputs = [self.GypPathToNinja(i, env) for i in action['inputs']] if int(action.get('process_outputs_as_sources', False)): @@ -616,15 +619,16 @@ def WriteActions(self, actions, extra_sources, prebuild, def WriteRules(self, rules, extra_sources, prebuild, mac_bundle_resources, extra_mac_bundle_resources): - env = self.GetSortedXcodeEnv() + env = self.GetToolchainEnv() all_outputs = [] for rule in rules: - # First write out a rule for the rule action. - name = '%s_%s' % (rule['rule_name'], - hashlib.md5(self.qualified_target).hexdigest()) # Skip a rule with no action and no inputs. if 'action' not in rule and not rule.get('rule_sources', []): continue + + # First write out a rule for the rule action. + name = '%s_%s' % (rule['rule_name'], self.hash_for_rules) + args = rule['action'] description = self.GenerateDescription( 'RULE', @@ -632,8 +636,9 @@ def WriteRules(self, rules, extra_sources, prebuild, ('%s ' + generator_default_variables['RULE_INPUT_PATH']) % name) is_cygwin = (self.msvs_settings.IsRuleRunUnderCygwin(rule) if self.flavor == 'win' else False) + pool = 'console' if int(rule.get('ninja_use_console', 0)) else None rule_name, args = self.WriteNewNinjaRule( - name, args, description, is_cygwin, env=env) + name, args, description, is_cygwin, env, pool) # TODO: if the command references the outputs directly, we should # simplify it to just use $out. @@ -645,16 +650,31 @@ def WriteRules(self, rules, extra_sources, prebuild, needed_variables = set(['source']) for argument in args: for var in special_locals: - if ('${%s}' % var) in argument: + if '${%s}' % var in argument: needed_variables.add(var) def cygwin_munge(path): + # pylint: disable=cell-var-from-loop if is_cygwin: return path.replace('\\', '/') return path + inputs = [self.GypPathToNinja(i, env) for i in rule.get('inputs', [])] + + # If there are n source files matching the rule, and m additional rule + # inputs, then adding 'inputs' to each build edge written below will + # write m * n inputs. Collapsing reduces this to m + n. + sources = rule.get('rule_sources', []) + num_inputs = len(inputs) + if prebuild: + num_inputs += 1 + if num_inputs > 2 and len(sources) > 2: + inputs = [self.WriteCollapsedDependencies( + rule['rule_name'], inputs, order_only=prebuild)] + prebuild = [] + # For each source file, write an edge that generates all the outputs. - for source in rule.get('rule_sources', []): + for source in sources: source = os.path.normpath(source) dirname, basename = os.path.split(source) root, ext = os.path.splitext(basename) @@ -663,9 +683,6 @@ def cygwin_munge(path): outputs = [self.ExpandRuleVariables(o, root, dirname, source, ext, basename) for o in rule['outputs']] - inputs = [self.ExpandRuleVariables(i, root, dirname, - source, ext, basename) - for i in rule.get('inputs', [])] if int(rule.get('process_outputs_as_sources', False)): extra_sources += outputs @@ -703,10 +720,11 @@ def cygwin_munge(path): else: assert var == None, repr(var) - inputs = [self.GypPathToNinja(i, env) for i in inputs] outputs = [self.GypPathToNinja(o, env) for o in outputs] - extra_bindings.append(('unique_name', - hashlib.md5(outputs[0]).hexdigest())) + if self.flavor == 'win': + # WriteNewNinjaRule uses unique_name for creating an rsp file on win. + extra_bindings.append(('unique_name', + hashlib.md5(outputs[0]).hexdigest())) self.ninja.build(outputs, rule_name, self.GypPathToNinja(source), implicit=inputs, order_only=prebuild, @@ -718,7 +736,7 @@ def cygwin_munge(path): def WriteCopies(self, copies, prebuild, mac_bundle_depends): outputs = [] - env = self.GetSortedXcodeEnv() + env = self.GetToolchainEnv() for copy in copies: for path in copy['files']: # Normalize the path so trailing slashes don't confuse us. @@ -742,15 +760,68 @@ def WriteCopies(self, copies, prebuild, mac_bundle_depends): def WriteMacBundleResources(self, resources, bundle_depends): """Writes ninja edges for 'mac_bundle_resources'.""" + xcassets = [] for output, res in gyp.xcode_emulation.GetMacBundleResources( generator_default_variables['PRODUCT_DIR'], self.xcode_settings, map(self.GypPathToNinja, resources)): output = self.ExpandSpecial(output) - self.ninja.build(output, 'mac_tool', res, - variables=[('mactool_cmd', 'copy-bundle-resource')]) - bundle_depends.append(output) + if os.path.splitext(output)[-1] != '.xcassets': + isBinary = self.xcode_settings.IsBinaryOutputFormat(self.config_name) + self.ninja.build(output, 'mac_tool', res, + variables=[('mactool_cmd', 'copy-bundle-resource'), \ + ('binary', isBinary)]) + bundle_depends.append(output) + else: + xcassets.append(res) + return xcassets + + def WriteMacXCassets(self, xcassets, bundle_depends): + """Writes ninja edges for 'mac_bundle_resources' .xcassets files. + + This add an invocation of 'actool' via the 'mac_tool.py' helper script. + It assumes that the assets catalogs define at least one imageset and + thus an Assets.car file will be generated in the application resources + directory. If this is not the case, then the build will probably be done + at each invocation of ninja.""" + if not xcassets: + return + + extra_arguments = {} + settings_to_arg = { + 'XCASSETS_APP_ICON': 'app-icon', + 'XCASSETS_LAUNCH_IMAGE': 'launch-image', + } + settings = self.xcode_settings.xcode_settings[self.config_name] + for settings_key, arg_name in settings_to_arg.iteritems(): + value = settings.get(settings_key) + if value: + extra_arguments[arg_name] = value + + partial_info_plist = None + if extra_arguments: + partial_info_plist = self.GypPathToUniqueOutput( + 'assetcatalog_generated_info.plist') + extra_arguments['output-partial-info-plist'] = partial_info_plist - def WriteMacInfoPlist(self, bundle_depends): + outputs = [] + outputs.append( + os.path.join( + self.xcode_settings.GetBundleResourceFolder(), + 'Assets.car')) + if partial_info_plist: + outputs.append(partial_info_plist) + + keys = QuoteShellArgument(json.dumps(extra_arguments), self.flavor) + extra_env = self.xcode_settings.GetPerTargetSettings() + env = self.GetSortedXcodeEnv(additional_settings=extra_env) + env = self.ComputeExportEnvString(env) + + bundle_depends.extend(self.ninja.build( + outputs, 'compile_xcassets', xcassets, + variables=[('env', env), ('keys', keys)])) + return partial_info_plist + + def WriteMacInfoPlist(self, partial_info_plist, bundle_depends): """Write build rules for bundle Info.plist files.""" info_plist, out, defines, extra_env = gyp.xcode_emulation.GetMacInfoPlist( generator_default_variables['PRODUCT_DIR'], @@ -770,10 +841,18 @@ def WriteMacInfoPlist(self, bundle_depends): env = self.GetSortedXcodeEnv(additional_settings=extra_env) env = self.ComputeExportEnvString(env) + if partial_info_plist: + intermediate_plist = self.GypPathToUniqueOutput('merged_info.plist') + info_plist = self.ninja.build( + intermediate_plist, 'merge_infoplist', + [partial_info_plist, info_plist]) + keys = self.xcode_settings.GetExtraPlistItems(self.config_name) keys = QuoteShellArgument(json.dumps(keys), self.flavor) + isBinary = self.xcode_settings.IsBinaryOutputFormat(self.config_name) self.ninja.build(out, 'copy_infoplist', info_plist, - variables=[('env', env), ('keys', keys)]) + variables=[('env', env), ('keys', keys), + ('binary', isBinary)]) bundle_depends.append(out) def WriteSources(self, ninja_file, config_name, config, sources, predepends, @@ -785,6 +864,8 @@ def WriteSources(self, ninja_file, config_name, config, sources, predepends, self.ninja.variable('cxx', '$cxx_host') self.ninja.variable('ld', '$ld_host') self.ninja.variable('ldxx', '$ldxx_host') + self.ninja.variable('nm', '$nm_host') + self.ninja.variable('readelf', '$readelf_host') if self.flavor != 'mac' or len(self.archs) == 1: return self.WriteSourcesForArch( @@ -810,6 +891,7 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, cflags_objcc = ['$cflags_cc'] + \ self.xcode_settings.GetCflagsObjCC(config_name) elif self.flavor == 'win': + asmflags = self.msvs_settings.GetAsmflags(config_name) cflags = self.msvs_settings.GetCflags(config_name) cflags_c = self.msvs_settings.GetCflagsC(config_name) cflags_cc = self.msvs_settings.GetCflagsCC(config_name) @@ -844,22 +926,31 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, self.WriteVariableList(ninja_file, 'defines', [Define(d, self.flavor) for d in defines]) if self.flavor == 'win': + self.WriteVariableList(ninja_file, 'asmflags', + map(self.ExpandSpecial, asmflags)) self.WriteVariableList(ninja_file, 'rcflags', [QuoteShellArgument(self.ExpandSpecial(f), self.flavor) for f in self.msvs_settings.GetRcflags(config_name, self.GypPathToNinja)]) include_dirs = config.get('include_dirs', []) - env = self.GetSortedXcodeEnv() + + env = self.GetToolchainEnv() if self.flavor == 'win': - env = self.msvs_settings.GetVSMacroEnv('$!PRODUCT_DIR', - config=config_name) include_dirs = self.msvs_settings.AdjustIncludeDirs(include_dirs, config_name) self.WriteVariableList(ninja_file, 'includes', [QuoteShellArgument('-I' + self.GypPathToNinja(i, env), self.flavor) for i in include_dirs]) + if self.flavor == 'win': + midl_include_dirs = config.get('midl_include_dirs', []) + midl_include_dirs = self.msvs_settings.AdjustMidlIncludeDirs( + midl_include_dirs, config_name) + self.WriteVariableList(ninja_file, 'midl_includes', + [QuoteShellArgument('-I' + self.GypPathToNinja(i, env), self.flavor) + for i in midl_include_dirs]) + pch_commands = precompiled_header.GetPchBuildCommands(arch) if self.flavor == 'mac': # Most targets use no precompiled headers, so only write these if needed. @@ -868,6 +959,8 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, include = precompiled_header.GetInclude(ext, arch) if include: ninja_file.variable(var, include) + arflags = config.get('arflags', []) + self.WriteVariableList(ninja_file, 'cflags', map(self.ExpandSpecial, cflags)) self.WriteVariableList(ninja_file, 'cflags_c', @@ -879,6 +972,8 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, map(self.ExpandSpecial, cflags_objc)) self.WriteVariableList(ninja_file, 'cflags_objcc', map(self.ExpandSpecial, cflags_objcc)) + self.WriteVariableList(ninja_file, 'arflags', + map(self.ExpandSpecial, arflags)) ninja_file.newline() outputs = [] has_rc_source = False @@ -894,9 +989,7 @@ def WriteSourcesForArch(self, ninja_file, config_name, config, sources, elif ext == 's' and self.flavor != 'win': # Doesn't generate .o.d files. command = 'cc_s' elif (self.flavor == 'win' and ext == 'asm' and - self.msvs_settings.GetArch(config_name) == 'x86' and not self.msvs_settings.HasExplicitAsmRules(spec)): - # Asm files only get auto assembled for x86 (not x64). command = 'asm' # Add the _asm suffix as msvs is capable of handling .cc and # .asm files of the same name without collision. @@ -968,9 +1061,19 @@ def WriteLink(self, spec, config_name, config, link_deps): arch=arch) for arch in self.archs] extra_bindings = [] + build_output = output if not self.is_mac_bundle: self.AppendPostbuildVariable(extra_bindings, spec, output, output) - self.ninja.build(output, 'lipo', inputs, variables=extra_bindings) + + # TODO(yyanagisawa): more work needed to fix: + # https://code.google.com/p/gyp/issues/detail?id=411 + if (spec['type'] in ('shared_library', 'loadable_module') and + not self.is_mac_bundle): + extra_bindings.append(('lib', output)) + self.ninja.build([output, output + '.TOC'], 'solipo', inputs, + variables=extra_bindings) + else: + self.ninja.build(build_output, 'lipo', inputs, variables=extra_bindings) return output def WriteLinkForArch(self, ninja_file, spec, config_name, config, @@ -1063,10 +1166,10 @@ def WriteLinkForArch(self, ninja_file, spec, config_name, config, rpath = 'lib/' if self.toolset != 'target': rpath += self.toolset - ldflags.append('-Wl,-rpath=\$$ORIGIN/%s' % rpath) + ldflags.append(r'-Wl,-rpath=\$$ORIGIN/%s' % rpath) ldflags.append('-Wl,-rpath-link=%s' % rpath) self.WriteVariableList(ninja_file, 'ldflags', - gyp.common.uniquer(map(self.ExpandSpecial, ldflags))) + map(self.ExpandSpecial, ldflags)) library_dirs = config.get('library_dirs', []) if self.flavor == 'win': @@ -1095,9 +1198,27 @@ def WriteLinkForArch(self, ninja_file, spec, config_name, config, extra_bindings.append(('soname', os.path.split(output)[1])) extra_bindings.append(('lib', gyp.common.EncodePOSIXShellArgument(output))) + if self.flavor != 'win': + link_file_list = output + if self.is_mac_bundle: + # 'Dependency Framework.framework/Versions/A/Dependency Framework' -> + # 'Dependency Framework.framework.rsp' + link_file_list = self.xcode_settings.GetWrapperName() + if arch: + link_file_list += '.' + arch + link_file_list += '.rsp' + # If an rspfile contains spaces, ninja surrounds the filename with + # quotes around it and then passes it to open(), creating a file with + # quotes in its name (and when looking for the rsp file, the name + # makes it through bash which strips the quotes) :-/ + link_file_list = link_file_list.replace(' ', '_') + extra_bindings.append( + ('link_file_list', + gyp.common.EncodePOSIXShellArgument(link_file_list))) if self.flavor == 'win': extra_bindings.append(('binary', output)) - if '/NOENTRY' not in ldflags: + if ('/NOENTRY' not in ldflags and + not self.msvs_settings.GetNoImportLibrary(config_name)): self.target.import_lib = output + '.lib' extra_bindings.append(('implibflag', '/IMPLIB:%s' % self.target.import_lib)) @@ -1196,6 +1317,19 @@ def WriteMacBundle(self, spec, mac_bundle_depends, is_empty): self.target.bundle = output return output + def GetToolchainEnv(self, additional_settings=None): + """Returns the variables toolchain would set for build steps.""" + env = self.GetSortedXcodeEnv(additional_settings=additional_settings) + if self.flavor == 'win': + env = self.GetMsvsToolchainEnv( + additional_settings=additional_settings) + return env + + def GetMsvsToolchainEnv(self, additional_settings=None): + """Returns the variables Visual Studio would set for build steps.""" + return self.msvs_settings.GetVSMacroEnv('$!PRODUCT_DIR', + config=self.config_name) + def GetSortedXcodeEnv(self, additional_settings=None): """Returns the variables Xcode would set for build steps.""" assert self.abs_build_dir @@ -1377,7 +1511,8 @@ def WriteVariableList(self, ninja_file, var, values): values = [] ninja_file.variable(var, ' '.join(values)) - def WriteNewNinjaRule(self, name, args, description, is_cygwin, env): + def WriteNewNinjaRule(self, name, args, description, is_cygwin, env, pool, + depfile=None): """Write out a new ninja "rule" statement for a given command. Returns the name of the new rule, and a copy of |args| with variables @@ -1435,7 +1570,8 @@ def WriteNewNinjaRule(self, name, args, description, is_cygwin, env): # GYP rules/actions express being no-ops by not touching their outputs. # Avoid executing downstream dependencies in this case by specifying # restat=1 to ninja. - self.ninja.rule(rule_name, command, description, restat=True, + self.ninja.rule(rule_name, command, description, depfile=depfile, + restat=True, pool=pool, rspfile=rspfile, rspfile_content=rspfile_content) self.ninja.newline() @@ -1466,12 +1602,13 @@ def CalculateVariables(default_variables, params): generator_extra_sources_for_rules = getattr(xcode_generator, 'generator_extra_sources_for_rules', []) elif flavor == 'win': + exts = gyp.MSVSUtil.TARGET_TYPE_EXT default_variables.setdefault('OS', 'win') - default_variables['EXECUTABLE_SUFFIX'] = '.exe' + default_variables['EXECUTABLE_SUFFIX'] = '.' + exts['executable'] default_variables['STATIC_LIB_PREFIX'] = '' - default_variables['STATIC_LIB_SUFFIX'] = '.lib' + default_variables['STATIC_LIB_SUFFIX'] = '.' + exts['static_library'] default_variables['SHARED_LIB_PREFIX'] = '' - default_variables['SHARED_LIB_SUFFIX'] = '.dll' + default_variables['SHARED_LIB_SUFFIX'] = '.' + exts['shared_library'] # Copy additional generator configuration data from VS, which is shared # by the Windows Ninja generator. @@ -1535,6 +1672,10 @@ def CommandWithWrapper(cmd, wrappers, prog): def GetDefaultConcurrentLinks(): """Returns a best-guess for a number of concurrent links.""" + pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) + if pool_size: + return pool_size + if sys.platform in ('win32', 'cygwin'): import ctypes @@ -1557,19 +1698,17 @@ class MEMORYSTATUSEX(ctypes.Structure): mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) - # return min(mem_limit, hard_cap) - # TODO(scottmg): Temporary speculative fix for OOM on builders - # See http://crbug.com/333000. - return 2 + return min(mem_limit, hard_cap) elif sys.platform.startswith('linux'): - with open("/proc/meminfo") as meminfo: - memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') - for line in meminfo: - match = memtotal_re.match(line) - if not match: - continue - # Allow 8Gb per link on Linux because Gold is quite memory hungry - return max(1, int(match.group(1)) / (8 * (2 ** 20))) + if os.path.exists("/proc/meminfo"): + with open("/proc/meminfo") as meminfo: + memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') + for line in meminfo: + match = memtotal_re.match(line) + if not match: + continue + # Allow 8Gb per link on Linux because Gold is quite memory hungry + return max(1, int(match.group(1)) / (8 * (2 ** 20))) return 1 elif sys.platform == 'darwin': try: @@ -1666,14 +1805,15 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, # 'CC_host'/'CXX_host' enviroment variable, cc_host/cxx_host should be set # to cc/cxx. if flavor == 'win': - # Overridden by local arch choice in the use_deps case. - # Chromium's ffmpeg c99conv.py currently looks for a 'cc =' line in - # build.ninja so needs something valid here. http://crbug.com/233985 - cc = 'cl.exe' - cxx = 'cl.exe' + ar = 'lib.exe' + # cc and cxx must be set to the correct architecture by overriding with one + # of cl_x86 or cl_x64 below. + cc = 'UNSET' + cxx = 'UNSET' ld = 'link.exe' ld_host = '$ld' else: + ar = 'ar' cc = 'cc' cxx = 'c++' ld = '$cc' @@ -1681,10 +1821,16 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, ld_host = '$cc_host' ldxx_host = '$cxx_host' + ar_host = 'ar' cc_host = None cxx_host = None cc_host_global_setting = None cxx_host_global_setting = None + clang_cl = None + nm = 'nm' + nm_host = 'nm' + readelf = 'readelf' + readelf_host = 'readelf' build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0]) make_global_settings = data[build_file].get('make_global_settings', []) @@ -1692,8 +1838,14 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, options.toplevel_dir) wrappers = {} for key, value in make_global_settings: + if key == 'AR': + ar = os.path.join(build_to_root, value) + if key == 'AR.host': + ar_host = os.path.join(build_to_root, value) if key == 'CC': cc = os.path.join(build_to_root, value) + if cc.endswith('clang-cl'): + clang_cl = cc if key == 'CXX': cxx = os.path.join(build_to_root, value) if key == 'CC.host': @@ -1702,6 +1854,18 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, if key == 'CXX.host': cxx_host = os.path.join(build_to_root, value) cxx_host_global_setting = value + if key == 'LD': + ld = os.path.join(build_to_root, value) + if key == 'LD.host': + ld_host = os.path.join(build_to_root, value) + if key == 'NM': + nm = os.path.join(build_to_root, value) + if key == 'NM.host': + nm_host = os.path.join(build_to_root, value) + if key == 'READELF': + readelf = os.path.join(build_to_root, value) + if key == 'READELF.host': + readelf_host = os.path.join(build_to_root, value) if key.endswith('_wrapper'): wrappers[key[:-len('_wrapper')]] = os.path.join(build_to_root, value) @@ -1713,12 +1877,25 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, wrappers[key_prefix] = os.path.join(build_to_root, value) if flavor == 'win': + configs = [target_dicts[qualified_target]['configurations'][config_name] + for qualified_target in target_list] + shared_system_includes = None + if not generator_flags.get('ninja_use_custom_environment_files', 0): + shared_system_includes = \ + gyp.msvs_emulation.ExtractSharedMSVSSystemIncludes( + configs, generator_flags) cl_paths = gyp.msvs_emulation.GenerateEnvironmentFiles( - toplevel_build, generator_flags, OpenOutput) + toplevel_build, generator_flags, shared_system_includes, OpenOutput) for arch, path in cl_paths.iteritems(): - master_ninja.variable( - 'cl_' + arch, CommandWithWrapper('CC', wrappers, - QuoteShellArgument(path, flavor))) + if clang_cl: + # If we have selected clang-cl, use that instead. + path = clang_cl + command = CommandWithWrapper('CC', wrappers, + QuoteShellArgument(path, 'win')) + if clang_cl: + # Use clang-cl to cross-compile for x86 or x86_64. + command += (' -m32' if arch == 'x86' else ' -m64') + master_ninja.variable('cl_' + arch, command) cc = GetEnvironFallback(['CC_target', 'CC'], cc) master_ninja.variable('cc', CommandWithWrapper('CC', wrappers, cc)) @@ -1728,14 +1905,22 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, if flavor == 'win': master_ninja.variable('ld', ld) master_ninja.variable('idl', 'midl.exe') - master_ninja.variable('ar', 'lib.exe') + master_ninja.variable('ar', ar) master_ninja.variable('rc', 'rc.exe') - master_ninja.variable('asm', 'ml.exe') + master_ninja.variable('ml_x86', 'ml.exe') + master_ninja.variable('ml_x64', 'ml64.exe') master_ninja.variable('mt', 'mt.exe') else: master_ninja.variable('ld', CommandWithWrapper('LINK', wrappers, ld)) master_ninja.variable('ldxx', CommandWithWrapper('LINK', wrappers, ldxx)) - master_ninja.variable('ar', GetEnvironFallback(['AR_target', 'AR'], 'ar')) + master_ninja.variable('ar', GetEnvironFallback(['AR_target', 'AR'], ar)) + if flavor != 'mac': + # Mac does not use readelf/nm for .TOC generation, so avoiding polluting + # the master ninja with extra unused variables. + master_ninja.variable( + 'nm', GetEnvironFallback(['NM_target', 'NM'], nm)) + master_ninja.variable( + 'readelf', GetEnvironFallback(['READELF_target', 'READELF'], readelf)) if generator_supports_multiple_toolsets: if not cc_host: @@ -1743,7 +1928,10 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, if not cxx_host: cxx_host = cxx - master_ninja.variable('ar_host', GetEnvironFallback(['AR_host'], 'ar')) + master_ninja.variable('ar_host', GetEnvironFallback(['AR_host'], ar_host)) + master_ninja.variable('nm_host', GetEnvironFallback(['NM_host'], nm_host)) + master_ninja.variable('readelf_host', + GetEnvironFallback(['READELF_host'], readelf_host)) cc_host = GetEnvironFallback(['CC_host'], cc_host) cxx_host = GetEnvironFallback(['CXX_host'], cxx_host) @@ -1826,7 +2014,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, description='IDL $in', command=('%s gyp-win-tool midl-wrapper $arch $outdir ' '$tlb $h $dlldata $iid $proxy $in ' - '$idlflags' % sys.executable)) + '$midl_includes $idlflags' % sys.executable)) master_ninja.rule( 'rc', description='RC $in', @@ -1836,20 +2024,20 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, sys.executable)) master_ninja.rule( 'asm', - description='ASM $in', + description='ASM $out', command=('%s gyp-win-tool asm-wrapper ' - '$arch $asm $defines $includes /c /Fo $out $in' % + '$arch $asm $defines $includes $asmflags /c /Fo $out $in' % sys.executable)) if flavor != 'mac' and flavor != 'win': master_ninja.rule( 'alink', description='AR $out', - command='rm -f $out && $ar rcs $out $in') + command='rm -f $out && $ar rcs $arflags $out $in') master_ninja.rule( 'alink_thin', description='AR $out', - command='rm -f $out && $ar rcsT $out $in') + command='rm -f $out && $ar rcsT $arflags $out $in') # This allows targets that only need to depend on $lib's API to declare an # order-only dependency on $lib.TOC and avoid relinking such downstream @@ -1857,38 +2045,39 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, # The resulting string leaves an uninterpolated %{suffix} which # is used in the final substitution below. mtime_preserving_solink_base = ( - 'if [ ! -e $lib -o ! -e ${lib}.TOC ]; then ' - '%(solink)s && %(extract_toc)s > ${lib}.TOC; else ' - '%(solink)s && %(extract_toc)s > ${lib}.tmp && ' - 'if ! cmp -s ${lib}.tmp ${lib}.TOC; then mv ${lib}.tmp ${lib}.TOC ; ' + 'if [ ! -e $lib -o ! -e $lib.TOC ]; then ' + '%(solink)s && %(extract_toc)s > $lib.TOC; else ' + '%(solink)s && %(extract_toc)s > $lib.tmp && ' + 'if ! cmp -s $lib.tmp $lib.TOC; then mv $lib.tmp $lib.TOC ; ' 'fi; fi' % { 'solink': '$ld -shared $ldflags -o $lib -Wl,-soname=$soname %(suffix)s', 'extract_toc': - ('{ readelf -d ${lib} | grep SONAME ; ' - 'nm -gD -f p ${lib} | cut -f1-2 -d\' \'; }')}) + ('{ $readelf -d $lib | grep SONAME ; ' + '$nm -gD -f p $lib | cut -f1-2 -d\' \'; }')}) master_ninja.rule( 'solink', description='SOLINK $lib', restat=True, - command=(mtime_preserving_solink_base % { - 'suffix': '-Wl,--whole-archive $in $solibs -Wl,--no-whole-archive ' - '$libs'}), + command=mtime_preserving_solink_base % {'suffix': '@$link_file_list'}, + rspfile='$link_file_list', + rspfile_content= + '-Wl,--whole-archive $in $solibs -Wl,--no-whole-archive $libs', pool='link_pool') master_ninja.rule( 'solink_module', description='SOLINK(module) $lib', restat=True, - command=(mtime_preserving_solink_base % { - 'suffix': '-Wl,--start-group $in $solibs -Wl,--end-group ' - '$libs'}), + command=mtime_preserving_solink_base % {'suffix': '@$link_file_list'}, + rspfile='$link_file_list', + rspfile_content='-Wl,--start-group $in -Wl,--end-group $solibs $libs', pool='link_pool') master_ninja.rule( 'link', description='LINK $out', command=('$ld $ldflags -o $out ' - '-Wl,--start-group $in $solibs -Wl,--end-group $libs'), + '-Wl,--start-group $in -Wl,--end-group $solibs $libs'), pool='link_pool') elif flavor == 'win': master_ninja.rule( @@ -1927,21 +2116,31 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, 'lipo', description='LIPO $out, POSTBUILDS', command='rm -f $out && lipo -create $in -output $out$postbuilds') + master_ninja.rule( + 'solipo', + description='SOLIPO $out, POSTBUILDS', + command=( + 'rm -f $lib $lib.TOC && lipo -create $in -output $lib$postbuilds &&' + '%(extract_toc)s > $lib.TOC' + % { 'extract_toc': + '{ otool -l $lib | grep LC_ID_DYLIB -A 5; ' + 'nm -gP $lib | cut -f1-2 -d\' \' | grep -v U$$; true; }'})) + # Record the public interface of $lib in $lib.TOC. See the corresponding # comment in the posix section above for details. solink_base = '$ld %(type)s $ldflags -o $lib %(suffix)s' mtime_preserving_solink_base = ( - 'if [ ! -e $lib -o ! -e ${lib}.TOC ] || ' + 'if [ ! -e $lib -o ! -e $lib.TOC ] || ' # Always force dependent targets to relink if this library # reexports something. Handling this correctly would require # recursive TOC dumping but this is rare in practice, so punt. 'otool -l $lib | grep -q LC_REEXPORT_DYLIB ; then ' - '%(solink)s && %(extract_toc)s > ${lib}.TOC; ' + '%(solink)s && %(extract_toc)s > $lib.TOC; ' 'else ' - '%(solink)s && %(extract_toc)s > ${lib}.tmp && ' - 'if ! cmp -s ${lib}.tmp ${lib}.TOC; then ' - 'mv ${lib}.tmp ${lib}.TOC ; ' + '%(solink)s && %(extract_toc)s > $lib.tmp && ' + 'if ! cmp -s $lib.tmp $lib.TOC; then ' + 'mv $lib.tmp $lib.TOC ; ' 'fi; ' 'fi' % { 'solink': solink_base, @@ -1949,34 +2148,42 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, '{ otool -l $lib | grep LC_ID_DYLIB -A 5; ' 'nm -gP $lib | cut -f1-2 -d\' \' | grep -v U$$; true; }'}) - solink_suffix = '$in $solibs $libs$postbuilds' + + solink_suffix = '@$link_file_list$postbuilds' master_ninja.rule( 'solink', description='SOLINK $lib, POSTBUILDS', restat=True, command=mtime_preserving_solink_base % {'suffix': solink_suffix, 'type': '-shared'}, + rspfile='$link_file_list', + rspfile_content='$in $solibs $libs', pool='link_pool') master_ninja.rule( 'solink_notoc', description='SOLINK $lib, POSTBUILDS', restat=True, command=solink_base % {'suffix':solink_suffix, 'type': '-shared'}, + rspfile='$link_file_list', + rspfile_content='$in $solibs $libs', pool='link_pool') - solink_module_suffix = '$in $solibs $libs$postbuilds' master_ninja.rule( 'solink_module', description='SOLINK(module) $lib, POSTBUILDS', restat=True, - command=mtime_preserving_solink_base % {'suffix': solink_module_suffix, + command=mtime_preserving_solink_base % {'suffix': solink_suffix, 'type': '-bundle'}, + rspfile='$link_file_list', + rspfile_content='$in $solibs $libs', pool='link_pool') master_ninja.rule( 'solink_module_notoc', description='SOLINK(module) $lib, POSTBUILDS', restat=True, - command=solink_base % {'suffix': solink_module_suffix, 'type': '-bundle'}, + command=solink_base % {'suffix': solink_suffix, 'type': '-bundle'}, + rspfile='$link_file_list', + rspfile_content='$in $solibs $libs', pool='link_pool') master_ninja.rule( @@ -1993,11 +2200,19 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, master_ninja.rule( 'copy_infoplist', description='COPY INFOPLIST $in', - command='$env ./gyp-mac-tool copy-info-plist $in $out $keys') + command='$env ./gyp-mac-tool copy-info-plist $in $out $binary $keys') + master_ninja.rule( + 'merge_infoplist', + description='MERGE INFOPLISTS $in', + command='$env ./gyp-mac-tool merge-info-plist $out $in') + master_ninja.rule( + 'compile_xcassets', + description='COMPILE XCASSETS $in', + command='$env ./gyp-mac-tool compile-xcassets $keys $in') master_ninja.rule( 'mac_tool', description='MACTOOL $mactool_cmd $in', - command='$env ./gyp-mac-tool $mactool_cmd $in $out') + command='$env ./gyp-mac-tool $mactool_cmd $in $out $binary') master_ninja.rule( 'package_framework', description='PACKAGE FRAMEWORK $out, POSTBUILDS', @@ -2037,6 +2252,15 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, # objects. target_short_names = {} + # short name of targets that were skipped because they didn't contain anything + # interesting. + # NOTE: there may be overlap between this an non_empty_target_names. + empty_target_names = set() + + # Set of non-empty short target names. + # NOTE: there may be overlap between this an empty_target_names. + non_empty_target_names = set() + for qualified_target in target_list: # qualified_target is like: third_party/icu/icu.gyp:icui18n#target build_file, name, toolset = \ @@ -2053,6 +2277,10 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, build_file = gyp.common.RelativePath(build_file, options.toplevel_dir) + qualified_target_for_hash = gyp.common.QualifiedTarget(build_file, name, + toolset) + hash_for_rules = hashlib.md5(qualified_target_for_hash).hexdigest() + base_path = os.path.dirname(build_file) obj = 'obj' if toolset != 'target': @@ -2060,7 +2288,7 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, output_file = os.path.join(obj, base_path, name + '.ninja') ninja_output = StringIO() - writer = NinjaWriter(qualified_target, target_outputs, base_path, build_dir, + writer = NinjaWriter(hash_for_rules, target_outputs, base_path, build_dir, ninja_output, toplevel_build, output_file, flavor, toplevel_dir=options.toplevel_dir) @@ -2080,6 +2308,9 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, target_outputs[qualified_target] = target if qualified_target in all_targets: all_outputs.add(target.FinalOutput()) + non_empty_target_names.add(name) + else: + empty_target_names.add(name) if target_short_names: # Write a short name to build this target. This benefits both the @@ -2091,6 +2322,16 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, master_ninja.build(short_name, 'phony', [x.FinalOutput() for x in target_short_names[short_name]]) + # Write phony targets for any empty targets that weren't written yet. As + # short names are not necessarily unique only do this for short names that + # haven't already been output for another target. + empty_target_names = empty_target_names - non_empty_target_names + if empty_target_names: + master_ninja.newline() + master_ninja.comment('Empty targets (output for completeness).') + for name in sorted(empty_target_names): + master_ninja.build(name, 'phony') + if all_outputs: master_ninja.newline() master_ninja.build('all', 'phony', list(all_outputs)) diff --git a/tools/gyp/pylib/gyp/generator/ninja_test.py b/tools/gyp/pylib/gyp/generator/ninja_test.py index 52661bcdf093..1767b2f45a04 100644 --- a/tools/gyp/pylib/gyp/generator/ninja_test.py +++ b/tools/gyp/pylib/gyp/generator/ninja_test.py @@ -15,15 +15,18 @@ class TestPrefixesAndSuffixes(unittest.TestCase): def test_BinaryNamesWindows(self): - writer = ninja.NinjaWriter('foo', 'wee', '.', '.', 'build.ninja', '.', - 'build.ninja', 'win') - spec = { 'target_name': 'wee' } - self.assertTrue(writer.ComputeOutputFileName(spec, 'executable'). - endswith('.exe')) - self.assertTrue(writer.ComputeOutputFileName(spec, 'shared_library'). - endswith('.dll')) - self.assertTrue(writer.ComputeOutputFileName(spec, 'static_library'). - endswith('.lib')) + # These cannot run on non-Windows as they require a VS installation to + # correctly handle variable expansion. + if sys.platform.startswith('win'): + writer = ninja.NinjaWriter('foo', 'wee', '.', '.', 'build.ninja', '.', + 'build.ninja', 'win') + spec = { 'target_name': 'wee' } + self.assertTrue(writer.ComputeOutputFileName(spec, 'executable'). + endswith('.exe')) + self.assertTrue(writer.ComputeOutputFileName(spec, 'shared_library'). + endswith('.dll')) + self.assertTrue(writer.ComputeOutputFileName(spec, 'static_library'). + endswith('.lib')) def test_BinaryNamesLinux(self): writer = ninja.NinjaWriter('foo', 'wee', '.', '.', 'build.ninja', '.', diff --git a/tools/gyp/pylib/gyp/generator/xcode.py b/tools/gyp/pylib/gyp/generator/xcode.py index 331e78baaa67..482b53ac8ad9 100644 --- a/tools/gyp/pylib/gyp/generator/xcode.py +++ b/tools/gyp/pylib/gyp/generator/xcode.py @@ -5,6 +5,7 @@ import filecmp import gyp.common import gyp.xcodeproj_file +import gyp.xcode_ninja import errno import os import sys @@ -68,6 +69,9 @@ # The Xcode-specific keys that exist on targets and aren't moved down to # configurations. generator_additional_non_configuration_keys = [ + 'ios_app_extension', + 'ios_watch_app', + 'ios_watchkit_extension', 'mac_bundle', 'mac_bundle_resources', 'mac_framework_headers', @@ -484,7 +488,7 @@ def Write(self): def AddSourceToTarget(source, type, pbxp, xct): # TODO(mark): Perhaps source_extensions and library_extensions can be made a # little bit fancier. - source_extensions = ['c', 'cc', 'cpp', 'cxx', 'm', 'mm', 's'] + source_extensions = ['c', 'cc', 'cpp', 'cxx', 'm', 'mm', 's', 'swift'] # .o is conceptually more of a "source" than a "library," but Xcode thinks # of "sources" as things to compile and "libraries" (or "frameworks") as @@ -520,7 +524,7 @@ def AddHeaderToTarget(header, pbxp, xct, is_public): xct.HeadersPhase().AddFile(header, settings) -_xcode_variable_re = re.compile('(\$\((.*?)\))') +_xcode_variable_re = re.compile(r'(\$\((.*?)\))') def ExpandXcodeVariables(string, expansions): """Expands Xcode-style $(VARIABLES) in string per the expansions dict. @@ -575,12 +579,17 @@ def PerformBuild(data, configurations, params): def GenerateOutput(target_list, target_dicts, data, params): + # Optionally configure each spec to use ninja as the external builder. + ninja_wrapper = params.get('flavor') == 'ninja' + if ninja_wrapper: + (target_list, target_dicts, data) = \ + gyp.xcode_ninja.CreateWrapper(target_list, target_dicts, data, params) + options = params['options'] generator_flags = params.get('generator_flags', {}) parallel_builds = generator_flags.get('xcode_parallel_builds', True) serialize_all_tests = \ generator_flags.get('xcode_serialize_all_test_runs', True) - project_version = generator_flags.get('xcode_project_version', None) skip_excluded_files = \ not generator_flags.get('xcode_list_excluded_files', True) xcode_projects = {} @@ -598,8 +607,6 @@ def GenerateOutput(target_list, target_dicts, data, params): if parallel_builds: pbxp.SetProperty('attributes', {'BuildIndependentTargetsInParallel': 'YES'}) - if project_version: - xcp.project_file.SetXcodeVersion(project_version) # Add gyp/gypi files to project if not generator_flags.get('standalone'): @@ -637,14 +644,18 @@ def GenerateOutput(target_list, target_dicts, data, params): # com.googlecode.gyp.xcode.bundle, a pseudo-type that xcode.py interprets # to create a single-file mh_bundle. _types = { - 'executable': 'com.apple.product-type.tool', - 'loadable_module': 'com.googlecode.gyp.xcode.bundle', - 'shared_library': 'com.apple.product-type.library.dynamic', - 'static_library': 'com.apple.product-type.library.static', - 'executable+bundle': 'com.apple.product-type.application', - 'loadable_module+bundle': 'com.apple.product-type.bundle', - 'loadable_module+xctest': 'com.apple.product-type.bundle.unit-test', - 'shared_library+bundle': 'com.apple.product-type.framework', + 'executable': 'com.apple.product-type.tool', + 'loadable_module': 'com.googlecode.gyp.xcode.bundle', + 'shared_library': 'com.apple.product-type.library.dynamic', + 'static_library': 'com.apple.product-type.library.static', + 'executable+bundle': 'com.apple.product-type.application', + 'loadable_module+bundle': 'com.apple.product-type.bundle', + 'loadable_module+xctest': 'com.apple.product-type.bundle.unit-test', + 'shared_library+bundle': 'com.apple.product-type.framework', + 'executable+extension+bundle': 'com.apple.product-type.app-extension', + 'executable+watch+extension+bundle': + 'com.apple.product-type.watchkit-extension', + 'executable+watch+bundle': 'com.apple.product-type.application.watchapp', } target_properties = { @@ -655,6 +666,9 @@ def GenerateOutput(target_list, target_dicts, data, params): type = spec['type'] is_xctest = int(spec.get('mac_xctest_bundle', 0)) is_bundle = int(spec.get('mac_bundle', 0)) or is_xctest + is_app_extension = int(spec.get('ios_app_extension', 0)) + is_watchkit_extension = int(spec.get('ios_watchkit_extension', 0)) + is_watch_app = int(spec.get('ios_watch_app', 0)) if type != 'none': type_bundle_key = type if is_xctest: @@ -662,6 +676,18 @@ def GenerateOutput(target_list, target_dicts, data, params): assert type == 'loadable_module', ( 'mac_xctest_bundle targets must have type loadable_module ' '(target %s)' % target_name) + elif is_app_extension: + assert is_bundle, ('ios_app_extension flag requires mac_bundle ' + '(target %s)' % target_name) + type_bundle_key += '+extension+bundle' + elif is_watchkit_extension: + assert is_bundle, ('ios_watchkit_extension flag requires mac_bundle ' + '(target %s)' % target_name) + type_bundle_key += '+watch+extension+bundle' + elif is_watch_app: + assert is_bundle, ('ios_watch_app flag requires mac_bundle ' + '(target %s)' % target_name) + type_bundle_key += '+watch+bundle' elif is_bundle: type_bundle_key += '+bundle' @@ -703,11 +729,16 @@ def GenerateOutput(target_list, target_dicts, data, params): # and is made a dependency of this target. This way the work is done # before the dependency checks for what should be recompiled. support_xct = None - if type != 'none' and (spec_actions or spec_rules): + # The Xcode "issues" don't affect xcode-ninja builds, since the dependency + # logic all happens in ninja. Don't bother creating the extra targets in + # that case. + if type != 'none' and (spec_actions or spec_rules) and not ninja_wrapper: support_xccl = CreateXCConfigurationList(configuration_names); + support_target_suffix = generator_flags.get( + 'support_target_suffix', ' Support') support_target_properties = { 'buildConfigurationList': support_xccl, - 'name': target_name + ' Support', + 'name': target_name + support_target_suffix, } if target_product_name: support_target_properties['productName'] = \ @@ -1096,6 +1127,9 @@ def GenerateOutput(target_list, target_dicts, data, params): # Relative paths are relative to $(SRCROOT). dest = '$(SRCROOT)/' + dest + code_sign = int(copy_group.get('xcode_code_sign', 0)) + settings = (None, '{ATTRIBUTES = (CodeSignOnCopy, ); }')[code_sign]; + # Coalesce multiple "copies" sections in the same target with the same # "destination" property into the same PBXCopyFilesBuildPhase, otherwise # they'll wind up with ID collisions. @@ -1114,7 +1148,7 @@ def GenerateOutput(target_list, target_dicts, data, params): pbxcp_dict[dest] = pbxcp for file in copy_group['files']: - pbxcp.AddFile(file) + pbxcp.AddFile(file, settings) # Excluded files can also go into the project file. if not skip_excluded_files: diff --git a/tools/gyp/pylib/gyp/input.py b/tools/gyp/pylib/gyp/input.py index 6472912db8eb..c7efe99a122c 100644 --- a/tools/gyp/pylib/gyp/input.py +++ b/tools/gyp/pylib/gyp/input.py @@ -10,8 +10,8 @@ from compiler.ast import Node from compiler.ast import Stmt import compiler -import copy import gyp.common +import gyp.simple_copy import multiprocessing import optparse import os.path @@ -24,6 +24,7 @@ import time import traceback from gyp.common import GypError +from gyp.common import OrderedSet # A list of types that are treated as linkable. @@ -45,18 +46,36 @@ 'outputs', 'sources', ] -path_sections = [] +path_sections = set() -is_path_section_charset = set('=+?!') -is_path_section_match_re = re.compile('_(dir|file|path)s?$') +# These per-process dictionaries are used to cache build file data when loading +# in parallel mode. +per_process_data = {} +per_process_aux_data = {} def IsPathSection(section): - # If section ends in one of these characters, it's applied to a section + # If section ends in one of the '=+?!' characters, it's applied to a section # without the trailing characters. '/' is notably absent from this list, # because there's no way for a regular expression to be treated as a path. - while section[-1:] in is_path_section_charset: + while section[-1:] in '=+?!': section = section[:-1] - return section in path_sections or is_path_section_match_re.search(section) + + if section in path_sections: + return True + + # Sections mathing the regexp '_(dir|file|path)s?$' are also + # considered PathSections. Using manual string matching since that + # is much faster than the regexp and this can be called hundreds of + # thousands of times so micro performance matters. + if "_" in section: + tail = section[-6:] + if tail[-1] == 's': + tail = tail[:-1] + if tail[-5:] in ('_file', '_path'): + return True + return tail[-4:] == '_dir' + + return False # base_non_configuration_keys is a list of key names that belong in the target # itself and should not be propagated into its configurations. It is merged @@ -196,11 +215,11 @@ def CheckNode(node, keypath): elif isinstance(node, Const): return node.getChildren()[0] else: - raise TypeError, "Unknown AST node at key path '" + '.'.join(keypath) + \ - "': " + repr(node) + raise TypeError("Unknown AST node at key path '" + '.'.join(keypath) + + "': " + repr(node)) -def LoadOneBuildFile(build_file_path, data, aux_data, variables, includes, +def LoadOneBuildFile(build_file_path, data, aux_data, includes, is_target, check): if build_file_path in data: return data[build_file_path] @@ -224,7 +243,7 @@ def LoadOneBuildFile(build_file_path, data, aux_data, variables, includes, gyp.common.ExceptionAppend(e, 'while reading ' + build_file_path) raise - if not isinstance(build_file_data, dict): + if type(build_file_data) is not dict: raise GypError("%s does not evaluate to a dictionary." % build_file_path) data[build_file_path] = build_file_data @@ -236,10 +255,10 @@ def LoadOneBuildFile(build_file_path, data, aux_data, variables, includes, try: if is_target: LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data, - aux_data, variables, includes, check) + aux_data, includes, check) else: LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data, - aux_data, variables, None, check) + aux_data, None, check) except Exception, e: gyp.common.ExceptionAppend(e, 'while reading includes of ' + build_file_path) @@ -249,7 +268,7 @@ def LoadOneBuildFile(build_file_path, data, aux_data, variables, includes, def LoadBuildFileIncludesIntoDict(subdict, subdict_path, data, aux_data, - variables, includes, check): + includes, check): includes_list = [] if includes != None: includes_list.extend(includes) @@ -273,30 +292,27 @@ def LoadBuildFileIncludesIntoDict(subdict, subdict_path, data, aux_data, gyp.DebugOutput(gyp.DEBUG_INCLUDES, "Loading Included File: '%s'", include) MergeDicts(subdict, - LoadOneBuildFile(include, data, aux_data, variables, None, - False, check), + LoadOneBuildFile(include, data, aux_data, None, False, check), subdict_path, include) # Recurse into subdictionaries. for k, v in subdict.iteritems(): - if v.__class__ == dict: - LoadBuildFileIncludesIntoDict(v, subdict_path, data, aux_data, variables, + if type(v) is dict: + LoadBuildFileIncludesIntoDict(v, subdict_path, data, aux_data, None, check) - elif v.__class__ == list: - LoadBuildFileIncludesIntoList(v, subdict_path, data, aux_data, variables, + elif type(v) is list: + LoadBuildFileIncludesIntoList(v, subdict_path, data, aux_data, check) # This recurses into lists so that it can look for dicts. -def LoadBuildFileIncludesIntoList(sublist, sublist_path, data, aux_data, - variables, check): +def LoadBuildFileIncludesIntoList(sublist, sublist_path, data, aux_data, check): for item in sublist: - if item.__class__ == dict: + if type(item) is dict: LoadBuildFileIncludesIntoDict(item, sublist_path, data, aux_data, - variables, None, check) - elif item.__class__ == list: - LoadBuildFileIncludesIntoList(item, sublist_path, data, aux_data, - variables, check) + None, check) + elif type(item) is list: + LoadBuildFileIncludesIntoList(item, sublist_path, data, aux_data, check) # Processes toolsets in all the targets. This recurses into condition entries # since they can contain toolsets as well. @@ -320,7 +336,7 @@ def ProcessToolsetsInDict(data): if len(toolsets) > 0: # Optimization: only do copies if more than one toolset is specified. for build in toolsets[1:]: - new_target = copy.deepcopy(target) + new_target = gyp.simple_copy.deepcopy(target) new_target['toolset'] = build new_target_list.append(new_target) target['toolset'] = toolsets[0] @@ -328,9 +344,10 @@ def ProcessToolsetsInDict(data): data['targets'] = new_target_list if 'conditions' in data: for condition in data['conditions']: - if isinstance(condition, list): + if type(condition) is list: for condition_dict in condition[1:]: - ProcessToolsetsInDict(condition_dict) + if type(condition_dict) is dict: + ProcessToolsetsInDict(condition_dict) # TODO(mark): I don't love this name. It just means that it's going to load @@ -350,15 +367,22 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, else: variables['DEPTH'] = d.replace('\\', '/') - if build_file_path in data['target_build_files']: - # Already loaded. - return False - data['target_build_files'].add(build_file_path) + # The 'target_build_files' key is only set when loading target build files in + # the non-parallel code path, where LoadTargetBuildFile is called + # recursively. In the parallel code path, we don't need to check whether the + # |build_file_path| has already been loaded, because the 'scheduled' set in + # ParallelState guarantees that we never load the same |build_file_path| + # twice. + if 'target_build_files' in data: + if build_file_path in data['target_build_files']: + # Already loaded. + return False + data['target_build_files'].add(build_file_path) gyp.DebugOutput(gyp.DEBUG_INCLUDES, "Loading Target Build File '%s'", build_file_path) - build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables, + build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, includes, True, check) # Store DEPTH for later use in generators. @@ -408,7 +432,8 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, # copy with the target-specific data merged into it as the replacement # target dict. old_target_dict = build_file_data['targets'][index] - new_target_dict = copy.deepcopy(build_file_data['target_defaults']) + new_target_dict = gyp.simple_copy.deepcopy( + build_file_data['target_defaults']) MergeDicts(new_target_dict, old_target_dict, build_file_path, build_file_path) build_file_data['targets'][index] = new_target_dict @@ -443,10 +468,8 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, else: return (build_file_path, dependencies) - def CallLoadTargetBuildFile(global_flags, - build_file_path, data, - aux_data, variables, + build_file_path, variables, includes, depth, check, generator_input_info): """Wrapper around LoadTargetBuildFile for parallel processing. @@ -462,35 +485,24 @@ def CallLoadTargetBuildFile(global_flags, for key, value in global_flags.iteritems(): globals()[key] = value - # Save the keys so we can return data that changed. - data_keys = set(data) - aux_data_keys = set(aux_data) - SetGeneratorGlobals(generator_input_info) - result = LoadTargetBuildFile(build_file_path, data, - aux_data, variables, + result = LoadTargetBuildFile(build_file_path, per_process_data, + per_process_aux_data, variables, includes, depth, check, False) if not result: return result (build_file_path, dependencies) = result - data_out = {} - for key in data: - if key == 'target_build_files': - continue - if key not in data_keys: - data_out[key] = data[key] - aux_data_out = {} - for key in aux_data: - if key not in aux_data_keys: - aux_data_out[key] = aux_data[key] + # We can safely pop the build_file_data from per_process_data because it + # will never be referenced by this process again, so we don't need to keep + # it in the cache. + build_file_data = per_process_data.pop(build_file_path) # This gets serialized and sent back to the main process via a pipe. # It's handled in LoadTargetBuildFileCallback. return (build_file_path, - data_out, - aux_data_out, + build_file_data, dependencies) except GypError, e: sys.stderr.write("gyp: %s\n" % e) @@ -521,8 +533,6 @@ def __init__(self): self.condition = None # The "data" dict that was passed to LoadTargetBuildFileParallel self.data = None - # The "aux_data" dict that was passed to LoadTargetBuildFileParallel - self.aux_data = None # The number of parallel calls outstanding; decremented when a response # was received. self.pending = 0 @@ -543,12 +553,9 @@ def LoadTargetBuildFileCallback(self, result): self.condition.notify() self.condition.release() return - (build_file_path0, data0, aux_data0, dependencies0) = result + (build_file_path0, build_file_data0, dependencies0) = result + self.data[build_file_path0] = build_file_data0 self.data['target_build_files'].add(build_file_path0) - for key in data0: - self.data[key] = data0[key] - for key in aux_data0: - self.aux_data[key] = aux_data0[key] for new_dependency in dependencies0: if new_dependency not in self.scheduled: self.scheduled.add(new_dependency) @@ -558,9 +565,8 @@ def LoadTargetBuildFileCallback(self, result): self.condition.release() -def LoadTargetBuildFilesParallel(build_files, data, aux_data, - variables, includes, depth, check, - generator_input_info): +def LoadTargetBuildFilesParallel(build_files, data, variables, includes, depth, + check, generator_input_info): parallel_state = ParallelState() parallel_state.condition = threading.Condition() # Make copies of the build_files argument that we can modify while working. @@ -568,7 +574,6 @@ def LoadTargetBuildFilesParallel(build_files, data, aux_data, parallel_state.scheduled = set(build_files) parallel_state.pending = 0 parallel_state.data = data - parallel_state.aux_data = aux_data try: parallel_state.condition.acquire() @@ -582,20 +587,16 @@ def LoadTargetBuildFilesParallel(build_files, data, aux_data, dependency = parallel_state.dependencies.pop() parallel_state.pending += 1 - data_in = {} - data_in['target_build_files'] = data['target_build_files'] - aux_data_in = {} global_flags = { 'path_sections': globals()['path_sections'], 'non_configuration_keys': globals()['non_configuration_keys'], 'multiple_toolsets': globals()['multiple_toolsets']} if not parallel_state.pool: - parallel_state.pool = multiprocessing.Pool(8) + parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count()) parallel_state.pool.apply_async( CallLoadTargetBuildFile, args = (global_flags, dependency, - data_in, aux_data_in, variables, includes, depth, check, generator_input_info), callback = parallel_state.LoadTargetBuildFileCallback) except KeyboardInterrupt, e: @@ -636,39 +637,50 @@ def FindEnclosingBracketGroup(input_str): return (-1, -1) -canonical_int_re = re.compile('(0|-?[1-9][0-9]*)$') - - def IsStrCanonicalInt(string): """Returns True if |string| is in its canonical integer form. The canonical form is such that str(int(string)) == string. """ - return isinstance(string, str) and canonical_int_re.match(string) + if type(string) is str: + # This function is called a lot so for maximum performance, avoid + # involving regexps which would otherwise make the code much + # shorter. Regexps would need twice the time of this function. + if string: + if string == "0": + return True + if string[0] == "-": + string = string[1:] + if not string: + return False + if '1' <= string[0] <= '9': + return string.isdigit() + + return False # This matches things like "<(asdf)", "(?P<(?:(?:!?@?)|\|)?)' - '(?P[-a-zA-Z0-9_.]+)?' - '\((?P\s*\[?)' - '(?P.*?)(\]?)\))') + r'(?P(?P<(?:(?:!?@?)|\|)?)' + r'(?P[-a-zA-Z0-9_.]+)?' + r'\((?P\s*\[?)' + r'(?P.*?)(\]?)\))') # This matches the same as early_variable_re, but with '>' instead of '<'. late_variable_re = re.compile( - '(?P(?P>(?:(?:!?@?)|\|)?)' - '(?P[-a-zA-Z0-9_.]+)?' - '\((?P\s*\[?)' - '(?P.*?)(\]?)\))') + r'(?P(?P>(?:(?:!?@?)|\|)?)' + r'(?P[-a-zA-Z0-9_.]+)?' + r'\((?P\s*\[?)' + r'(?P.*?)(\]?)\))') # This matches the same as early_variable_re, but with '^' instead of '<'. latelate_variable_re = re.compile( - '(?P(?P[\^](?:(?:!?@?)|\|)?)' - '(?P[-a-zA-Z0-9_.]+)?' - '\((?P\s*\[?)' - '(?P.*?)(\]?)\))') + r'(?P(?P[\^](?:(?:!?@?)|\|)?)' + r'(?P[-a-zA-Z0-9_.]+)?' + r'\((?P\s*\[?)' + r'(?P.*?)(\]?)\))') # Global cache of results from running commands so they don't have to be run # more then once. @@ -677,7 +689,7 @@ def IsStrCanonicalInt(string): def FixupPlatformCommand(cmd): if sys.platform == 'win32': - if type(cmd) == list: + if type(cmd) is list: cmd = [re.sub('^cat ', 'type ', cmd[0])] + cmd[1:] else: cmd = re.sub('^cat ', 'type ', cmd) @@ -767,7 +779,7 @@ def ExpandVariables(input, phase, variables, build_file): # contexts. However, since filtration has no chance to run on <|(), # this seems like the only obvious way to give them access to filters. if file_list: - processed_variables = copy.deepcopy(variables) + processed_variables = gyp.simple_copy.deepcopy(variables) ProcessListFiltersInDict(contents, processed_variables) # Recurse to expand variables in the contents contents = ExpandVariables(contents, phase, @@ -804,7 +816,7 @@ def ExpandVariables(input, phase, variables, build_file): # This works around actions/rules which have more inputs than will # fit on the command line. if file_list: - if type(contents) == list: + if type(contents) is list: contents_list = contents else: contents_list = contents.split(' ') @@ -837,17 +849,15 @@ def ExpandVariables(input, phase, variables, build_file): use_shell = False # Check for a cached value to avoid executing commands, or generating - # file lists more than once. - # TODO(http://code.google.com/p/gyp/issues/detail?id=112): It is - # possible that the command being invoked depends on the current - # directory. For that case the syntax needs to be extended so that the - # directory is also used in cache_key (it becomes a tuple). + # file lists more than once. The cache key contains the command to be + # run as well as the directory to run it from, to account for commands + # that depend on their current directory. # TODO(http://code.google.com/p/gyp/issues/detail?id=111): In theory, # someone could author a set of GYP files where each time the command # is invoked it produces different output by design. When the need # arises, the syntax should be extended to support no caching off a # command's output so it is run every time. - cache_key = str(contents) + cache_key = (str(contents), build_file_dir) cached_value = cached_command_results.get(cache_key, None) if cached_value is None: gyp.DebugOutput(gyp.DEBUG_VARIABLES, @@ -925,10 +935,9 @@ def ExpandVariables(input, phase, variables, build_file): else: replacement = variables[contents] - if isinstance(replacement, list): + if type(replacement) is list: for item in replacement: - if (not contents[-1] == '/' and - not isinstance(item, str) and not isinstance(item, int)): + if not contents[-1] == '/' and type(item) not in (str, int): raise GypError('Variable ' + contents + ' must expand to a string or list of strings; ' + 'list contains a ' + @@ -938,8 +947,7 @@ def ExpandVariables(input, phase, variables, build_file): # with conditions sections. ProcessVariablesAndConditionsInList(replacement, phase, variables, build_file) - elif not isinstance(replacement, str) and \ - not isinstance(replacement, int): + elif type(replacement) not in (str, int): raise GypError('Variable ' + contents + ' must expand to a string or list of strings; ' + 'found a ' + replacement.__class__.__name__) @@ -948,7 +956,7 @@ def ExpandVariables(input, phase, variables, build_file): # Expanding in list context. It's guaranteed that there's only one # replacement to do in |input_str| and that it's this replacement. See # above. - if isinstance(replacement, list): + if type(replacement) is list: # If it's already a list, make a copy. output = replacement[:] else: @@ -957,7 +965,7 @@ def ExpandVariables(input, phase, variables, build_file): else: # Expanding in string context. encoded_replacement = '' - if isinstance(replacement, list): + if type(replacement) is list: # When expanding a list into string context, turn the list items # into a string in a way that will work with a subprocess call. # @@ -975,26 +983,32 @@ def ExpandVariables(input, phase, variables, build_file): # Prepare for the next match iteration. input_str = output - # Look for more matches now that we've replaced some, to deal with - # expanding local variables (variables defined in the same - # variables block as this one). - gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Found output %r, recursing.", output) - if isinstance(output, list): - if output and isinstance(output[0], list): - # Leave output alone if it's a list of lists. - # We don't want such lists to be stringified. - pass - else: - new_output = [] - for item in output: - new_output.append( - ExpandVariables(item, phase, variables, build_file)) - output = new_output + if output == input: + gyp.DebugOutput(gyp.DEBUG_VARIABLES, + "Found only identity matches on %r, avoiding infinite " + "recursion.", + output) else: - output = ExpandVariables(output, phase, variables, build_file) + # Look for more matches now that we've replaced some, to deal with + # expanding local variables (variables defined in the same + # variables block as this one). + gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Found output %r, recursing.", output) + if type(output) is list: + if output and type(output[0]) is list: + # Leave output alone if it's a list of lists. + # We don't want such lists to be stringified. + pass + else: + new_output = [] + for item in output: + new_output.append( + ExpandVariables(item, phase, variables, build_file)) + output = new_output + else: + output = ExpandVariables(output, phase, variables, build_file) # Convert all strings that are canonically-represented integers into integers. - if isinstance(output, list): + if type(output) is list: for index in xrange(0, len(output)): if IsStrCanonicalInt(output[index]): output[index] = int(output[index]) @@ -1003,6 +1017,80 @@ def ExpandVariables(input, phase, variables, build_file): return output +# The same condition is often evaluated over and over again so it +# makes sense to cache as much as possible between evaluations. +cached_conditions_asts = {} + +def EvalCondition(condition, conditions_key, phase, variables, build_file): + """Returns the dict that should be used or None if the result was + that nothing should be used.""" + if type(condition) is not list: + raise GypError(conditions_key + ' must be a list') + if len(condition) < 2: + # It's possible that condition[0] won't work in which case this + # attempt will raise its own IndexError. That's probably fine. + raise GypError(conditions_key + ' ' + condition[0] + + ' must be at least length 2, not ' + str(len(condition))) + + i = 0 + result = None + while i < len(condition): + cond_expr = condition[i] + true_dict = condition[i + 1] + if type(true_dict) is not dict: + raise GypError('{} {} must be followed by a dictionary, not {}'.format( + conditions_key, cond_expr, type(true_dict))) + if len(condition) > i + 2 and type(condition[i + 2]) is dict: + false_dict = condition[i + 2] + i = i + 3 + if i != len(condition): + raise GypError('{} {} has {} unexpected trailing items'.format( + conditions_key, cond_expr, len(condition) - i)) + else: + false_dict = None + i = i + 2 + if result == None: + result = EvalSingleCondition( + cond_expr, true_dict, false_dict, phase, variables, build_file) + + return result + + +def EvalSingleCondition( + cond_expr, true_dict, false_dict, phase, variables, build_file): + """Returns true_dict if cond_expr evaluates to true, and false_dict + otherwise.""" + # Do expansions on the condition itself. Since the conditon can naturally + # contain variable references without needing to resort to GYP expansion + # syntax, this is of dubious value for variables, but someone might want to + # use a command expansion directly inside a condition. + cond_expr_expanded = ExpandVariables(cond_expr, phase, variables, + build_file) + if type(cond_expr_expanded) not in (str, int): + raise ValueError( + 'Variable expansion in this context permits str and int ' + \ + 'only, found ' + cond_expr_expanded.__class__.__name__) + + try: + if cond_expr_expanded in cached_conditions_asts: + ast_code = cached_conditions_asts[cond_expr_expanded] + else: + ast_code = compile(cond_expr_expanded, '', 'eval') + cached_conditions_asts[cond_expr_expanded] = ast_code + if eval(ast_code, {'__builtins__': None}, variables): + return true_dict + return false_dict + except SyntaxError, e: + syntax_error = SyntaxError('%s while evaluating condition \'%s\' in %s ' + 'at character %d.' % + (str(e.args[0]), e.text, build_file, e.offset), + e.filename, e.lineno, e.offset, e.text) + raise syntax_error + except NameError, e: + gyp.common.ExceptionAppend(e, 'while evaluating condition \'%s\' in %s' % + (cond_expr_expanded, build_file)) + raise GypError(e) + def ProcessConditionsInDict(the_dict, phase, variables, build_file): # Process a 'conditions' or 'target_conditions' section in the_dict, @@ -1038,48 +1126,8 @@ def ProcessConditionsInDict(the_dict, phase, variables, build_file): del the_dict[conditions_key] for condition in conditions_list: - if not isinstance(condition, list): - raise GypError(conditions_key + ' must be a list') - if len(condition) != 2 and len(condition) != 3: - # It's possible that condition[0] won't work in which case this - # attempt will raise its own IndexError. That's probably fine. - raise GypError(conditions_key + ' ' + condition[0] + - ' must be length 2 or 3, not ' + str(len(condition))) - - [cond_expr, true_dict] = condition[0:2] - false_dict = None - if len(condition) == 3: - false_dict = condition[2] - - # Do expansions on the condition itself. Since the conditon can naturally - # contain variable references without needing to resort to GYP expansion - # syntax, this is of dubious value for variables, but someone might want to - # use a command expansion directly inside a condition. - cond_expr_expanded = ExpandVariables(cond_expr, phase, variables, - build_file) - if not isinstance(cond_expr_expanded, str) and \ - not isinstance(cond_expr_expanded, int): - raise ValueError, \ - 'Variable expansion in this context permits str and int ' + \ - 'only, found ' + expanded.__class__.__name__ - - try: - ast_code = compile(cond_expr_expanded, '', 'eval') - - if eval(ast_code, {'__builtins__': None}, variables): - merge_dict = true_dict - else: - merge_dict = false_dict - except SyntaxError, e: - syntax_error = SyntaxError('%s while evaluating condition \'%s\' in %s ' - 'at character %d.' % - (str(e.args[0]), e.text, build_file, e.offset), - e.filename, e.lineno, e.offset, e.text) - raise syntax_error - except NameError, e: - gyp.common.ExceptionAppend(e, 'while evaluating condition \'%s\' in %s' % - (cond_expr_expanded, build_file)) - raise GypError(e) + merge_dict = EvalCondition(condition, conditions_key, phase, variables, + build_file) if merge_dict != None: # Expand variables and nested conditinals in the merge_dict before @@ -1094,8 +1142,7 @@ def LoadAutomaticVariablesFromDict(variables, the_dict): # Any keys with plain string values in the_dict become automatic variables. # The variable name is the key name with a "_" character prepended. for key, value in the_dict.iteritems(): - if isinstance(value, str) or isinstance(value, int) or \ - isinstance(value, list): + if type(value) in (str, int, list): variables['_' + key] = value @@ -1108,8 +1155,7 @@ def LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key): # (it could be a list or it could be parentless because it is a root dict), # the_dict_key will be None. for key, value in the_dict.get('variables', {}).iteritems(): - if not isinstance(value, str) and not isinstance(value, int) and \ - not isinstance(value, list): + if type(value) not in (str, int, list): continue if key.endswith('%'): @@ -1162,12 +1208,12 @@ def ProcessVariablesAndConditionsInDict(the_dict, phase, variables_in, for key, value in the_dict.iteritems(): # Skip "variables", which was already processed if present. - if key != 'variables' and isinstance(value, str): + if key != 'variables' and type(value) is str: expanded = ExpandVariables(value, phase, variables, build_file) - if not isinstance(expanded, str) and not isinstance(expanded, int): - raise ValueError, \ + if type(expanded) not in (str, int): + raise ValueError( 'Variable expansion in this context permits str and int ' + \ - 'only, found ' + expanded.__class__.__name__ + ' for ' + key + 'only, found ' + expanded.__class__.__name__ + ' for ' + key) the_dict[key] = expanded # Variable expansion may have resulted in changes to automatics. Reload. @@ -1221,23 +1267,23 @@ def ProcessVariablesAndConditionsInDict(the_dict, phase, variables_in, for key, value in the_dict.iteritems(): # Skip "variables" and string values, which were already processed if # present. - if key == 'variables' or isinstance(value, str): + if key == 'variables' or type(value) is str: continue - if isinstance(value, dict): + if type(value) is dict: # Pass a copy of the variables dict so that subdicts can't influence # parents. ProcessVariablesAndConditionsInDict(value, phase, variables, build_file, key) - elif isinstance(value, list): + elif type(value) is list: # The list itself can't influence the variables dict, and # ProcessVariablesAndConditionsInList will make copies of the variables # dict if it needs to pass it to something that can influence it. No # copy is necessary here. ProcessVariablesAndConditionsInList(value, phase, variables, build_file) - elif not isinstance(value, int): - raise TypeError, 'Unknown type ' + value.__class__.__name__ + \ - ' for ' + key + elif type(value) is not int: + raise TypeError('Unknown type ' + value.__class__.__name__ + \ + ' for ' + key) def ProcessVariablesAndConditionsInList(the_list, phase, variables, @@ -1246,17 +1292,17 @@ def ProcessVariablesAndConditionsInList(the_list, phase, variables, index = 0 while index < len(the_list): item = the_list[index] - if isinstance(item, dict): + if type(item) is dict: # Make a copy of the variables dict so that it won't influence anything # outside of its own scope. ProcessVariablesAndConditionsInDict(item, phase, variables, build_file) - elif isinstance(item, list): + elif type(item) is list: ProcessVariablesAndConditionsInList(item, phase, variables, build_file) - elif isinstance(item, str): + elif type(item) is str: expanded = ExpandVariables(item, phase, variables, build_file) - if isinstance(expanded, str) or isinstance(expanded, int): + if type(expanded) in (str, int): the_list[index] = expanded - elif isinstance(expanded, list): + elif type(expanded) is list: the_list[index:index+1] = expanded index += len(expanded) @@ -1264,13 +1310,13 @@ def ProcessVariablesAndConditionsInList(the_list, phase, variables, # without falling into the index increment below. continue else: - raise ValueError, \ + raise ValueError( 'Variable expansion in this context permits strings and ' + \ 'lists only, found ' + expanded.__class__.__name__ + ' at ' + \ - index - elif not isinstance(item, int): - raise TypeError, 'Unknown type ' + item.__class__.__name__ + \ - ' at index ' + index + index) + elif type(item) is not int: + raise TypeError('Unknown type ' + item.__class__.__name__ + \ + ' at index ' + index) index = index + 1 @@ -1443,6 +1489,20 @@ def RemoveSelfDependencies(targets): target_dict[dependency_key] = Filter(dependencies, target_name) +def RemoveLinkDependenciesFromNoneTargets(targets): + """Remove dependencies having the 'link_dependency' attribute from the 'none' + targets.""" + for target_name, target_dict in targets.iteritems(): + for dependency_key in dependency_sections: + dependencies = target_dict.get(dependency_key, []) + if dependencies: + for t in dependencies: + if target_dict.get('type', None) == 'none': + if targets[t].get('variables', {}).get('link_dependency', 0): + target_dict[dependency_key] = \ + Filter(target_dict[dependency_key], t) + + class DependencyGraphNode(object): """ @@ -1468,7 +1528,7 @@ def FlattenToList(self): # are the "ref" attributes of DependencyGraphNodes. Every target will # appear in flat_list after all of its dependencies, and before all of its # dependents. - flat_list = [] + flat_list = OrderedSet() # in_degree_zeros is the list of DependencyGraphNodes that have no # dependencies not in flat_list. Initially, it is a copy of the children @@ -1482,12 +1542,15 @@ def FlattenToList(self): # as work progresses, so that the next node to process from the list can # always be accessed at a consistent position. node = in_degree_zeros.pop() - flat_list.append(node.ref) + flat_list.add(node.ref) # Look at dependents of the node just added to flat_list. Some of them # may now belong in in_degree_zeros. for node_dependent in node.dependents: is_in_degree_zero = True + # TODO: We want to check through the + # node_dependent.dependencies list but if it's long and we + # always start at the beginning, then we get O(n^2) behaviour. for node_dependent_dependency in node_dependent.dependencies: if not node_dependent_dependency.ref in flat_list: # The dependent one or more dependencies not in flat_list. There @@ -1503,28 +1566,27 @@ def FlattenToList(self): # iteration of the outer loop. in_degree_zeros.add(node_dependent) - return flat_list + return list(flat_list) - def FindCycles(self, path=None): + def FindCycles(self): """ Returns a list of cycles in the graph, where each cycle is its own list. """ - if path is None: - path = [self] - results = [] - for node in self.dependents: - if node in path: - cycle = [node] - for part in path: - cycle.append(part) - if part == node: - break - results.append(tuple(cycle)) - else: - results.extend(node.FindCycles([node] + path)) + visited = set() + + def Visit(node, path): + for child in node.dependents: + if child in path: + results.append([child] + path[:path.index(child) + 1]) + elif not child in visited: + visited.add(child) + Visit(child, [child] + path) - return list(set(results)) + visited.add(self) + Visit(self, [self]) + + return results def DirectDependencies(self, dependencies=None): """Returns a list of just direct dependencies.""" @@ -1589,21 +1651,26 @@ def DirectAndImportedDependencies(self, targets, dependencies=None): return self._AddImportedDependencies(targets, dependencies) def DeepDependencies(self, dependencies=None): - """Returns a list of all of a target's dependencies, recursively.""" - if dependencies == None: - dependencies = [] + """Returns an OrderedSet of all of a target's dependencies, recursively.""" + if dependencies is None: + # Using a list to get ordered output and a set to do fast "is it + # already added" checks. + dependencies = OrderedSet() for dependency in self.dependencies: # Check for None, corresponding to the root node. - if dependency.ref != None and dependency.ref not in dependencies: - dependencies.append(dependency.ref) + if dependency.ref is None: + continue + if dependency.ref not in dependencies: dependency.DeepDependencies(dependencies) + dependencies.add(dependency.ref) return dependencies def _LinkDependenciesInternal(self, targets, include_shared_libraries, dependencies=None, initial=True): - """Returns a list of dependency targets that are linked into this target. + """Returns an OrderedSet of dependency targets that are linked + into this target. This function has a split personality, depending on the setting of |initial|. Outside callers should always leave |initial| at its default @@ -1616,11 +1683,13 @@ def _LinkDependenciesInternal(self, targets, include_shared_libraries, If |include_shared_libraries| is False, the resulting dependencies will not include shared_library targets that are linked into this target. """ - if dependencies == None: - dependencies = [] + if dependencies is None: + # Using a list to get ordered output and a set to do fast "is it + # already added" checks. + dependencies = OrderedSet() # Check for None, corresponding to the root node. - if self.ref == None: + if self.ref is None: return dependencies # It's kind of sucky that |targets| has to be passed into this function, @@ -1648,8 +1717,7 @@ def _LinkDependenciesInternal(self, targets, include_shared_libraries, # Don't traverse 'none' targets if explicitly excluded. if (target_type == 'none' and not targets[self.ref].get('dependencies_traverse', True)): - if self.ref not in dependencies: - dependencies.append(self.ref) + dependencies.add(self.ref) return dependencies # Executables and loadable modules are already fully and finally linked. @@ -1671,7 +1739,7 @@ def _LinkDependenciesInternal(self, targets, include_shared_libraries, # The target is linkable, add it to the list of link dependencies. if self.ref not in dependencies: - dependencies.append(self.ref) + dependencies.add(self.ref) if initial or not is_linkable: # If this is a subsequent target and it's linkable, don't look any # further for linkable dependencies, as they'll already be linked into @@ -1735,12 +1803,22 @@ def BuildDependencyList(targets): flat_list = root_node.FlattenToList() # If there's anything left unvisited, there must be a circular dependency - # (cycle). If you need to figure out what's wrong, look for elements of - # targets that are not in flat_list. + # (cycle). if len(flat_list) != len(targets): + if not root_node.dependents: + # If all targets have dependencies, add the first target as a dependent + # of root_node so that the cycle can be discovered from root_node. + target = targets.keys()[0] + target_node = dependency_nodes[target] + target_node.dependencies.append(root_node) + root_node.dependents.append(target_node) + + cycles = [] + for cycle in root_node.FindCycles(): + paths = [node.ref for node in cycle] + cycles.append('Cycle: %s' % ' -> '.join(paths)) raise DependencyGraphNode.CircularException( - 'Some targets not reachable, cycle in dependency graph detected: ' + - ' '.join(set(flat_list) ^ set(targets))) + 'Cycles in dependency graph detected:\n' + '\n'.join(cycles)) return [dependency_nodes, flat_list] @@ -1790,20 +1868,18 @@ def VerifyNoGYPFileCircularDependencies(targets): # If there's anything left unvisited, there must be a circular dependency # (cycle). if len(flat_list) != len(dependency_nodes): - bad_files = [] - for file in dependency_nodes.iterkeys(): - if not file in flat_list: - bad_files.append(file) - common_path_prefix = os.path.commonprefix(dependency_nodes) + if not root_node.dependents: + # If all files have dependencies, add the first file as a dependent + # of root_node so that the cycle can be discovered from root_node. + file_node = dependency_nodes.values()[0] + file_node.dependencies.append(root_node) + root_node.dependents.append(file_node) cycles = [] for cycle in root_node.FindCycles(): - simplified_paths = [] - for node in cycle: - assert(node.ref.startswith(common_path_prefix)) - simplified_paths.append(node.ref[len(common_path_prefix):]) - cycles.append('Cycle: %s' % ' -> '.join(simplified_paths)) - raise DependencyGraphNode.CircularException, \ - 'Cycles in .gyp file dependency graph detected:\n' + '\n'.join(cycles) + paths = [node.ref for node in cycle] + cycles.append('Cycle: %s' % ' -> '.join(paths)) + raise DependencyGraphNode.CircularException( + 'Cycles in .gyp file dependency graph detected:\n' + '\n'.join(cycles)) def DoDependentSettings(key, flat_list, targets, dependency_nodes): @@ -1966,25 +2042,25 @@ def is_in_set_or_list(x, s, l): hashable_to_set = set(x for x in to if is_hashable(x)) for item in fro: singleton = False - if isinstance(item, str) or isinstance(item, int): + if type(item) in (str, int): # The cheap and easy case. if is_paths: to_item = MakePathRelative(to_file, fro_file, item) else: to_item = item - if not isinstance(item, str) or not item.startswith('-'): + if not (type(item) is str and item.startswith('-')): # Any string that doesn't begin with a "-" is a singleton - it can # only appear once in a list, to be enforced by the list merge append # or prepend. singleton = True - elif isinstance(item, dict): + elif type(item) is dict: # Make a copy of the dictionary, continuing to look for paths to fix. # The other intelligent aspects of merge processing won't apply because # item is being merged into an empty dict. to_item = {} MergeDicts(to_item, item, to_file, fro_file) - elif isinstance(item, list): + elif type(item) is list: # Recurse, making a copy of the list. If the list contains any # descendant dicts, path fixing will occur. Note that here, custom # values for is_paths and append are dropped; those are only to be @@ -1993,9 +2069,9 @@ def is_in_set_or_list(x, s, l): to_item = [] MergeLists(to_item, item, to_file, fro_file) else: - raise TypeError, \ + raise TypeError( 'Attempt to merge list item of unsupported type ' + \ - item.__class__.__name__ + item.__class__.__name__) if append: # If appending a singleton that's already in the list, don't append. @@ -2030,30 +2106,30 @@ def MergeDicts(to, fro, to_file, fro_file): # modified. if k in to: bad_merge = False - if isinstance(v, str) or isinstance(v, int): - if not (isinstance(to[k], str) or isinstance(to[k], int)): + if type(v) in (str, int): + if type(to[k]) not in (str, int): bad_merge = True - elif v.__class__ != to[k].__class__: + elif type(v) is not type(to[k]): bad_merge = True if bad_merge: - raise TypeError, \ + raise TypeError( 'Attempt to merge dict value of type ' + v.__class__.__name__ + \ ' into incompatible type ' + to[k].__class__.__name__ + \ - ' for key ' + k - if isinstance(v, str) or isinstance(v, int): + ' for key ' + k) + if type(v) in (str, int): # Overwrite the existing value, if any. Cheap and easy. is_path = IsPathSection(k) if is_path: to[k] = MakePathRelative(to_file, fro_file, v) else: to[k] = v - elif isinstance(v, dict): + elif type(v) is dict: # Recurse, guaranteeing copies will be made of objects that require it. if not k in to: to[k] = {} MergeDicts(to[k], v, to_file, fro_file) - elif isinstance(v, list): + elif type(v) is list: # Lists in dicts can be merged with different policies, depending on # how the key in the "from" dict (k, the from-key) is written. # @@ -2096,13 +2172,13 @@ def MergeDicts(to, fro, to_file, fro_file): # If the key ends in "?", the list will only be merged if it doesn't # already exist. continue - if not isinstance(to[list_base], list): + elif type(to[list_base]) is not list: # This may not have been checked above if merging in a list with an # extension character. - raise TypeError, \ + raise TypeError( 'Attempt to merge dict value of type ' + v.__class__.__name__ + \ ' into incompatible type ' + to[list_base].__class__.__name__ + \ - ' for key ' + list_base + '(' + k + ')' + ' for key ' + list_base + '(' + k + ')') else: to[list_base] = [] @@ -2114,9 +2190,9 @@ def MergeDicts(to, fro, to_file, fro_file): is_paths = IsPathSection(list_base) MergeLists(to[list_base], v, to_file, fro_file, is_paths, append) else: - raise TypeError, \ + raise TypeError( 'Attempt to merge dict value of unsupported type ' + \ - v.__class__.__name__ + ' for key ' + k + v.__class__.__name__ + ' for key ' + k) def MergeConfigWithInheritance(new_configuration_dict, build_file, @@ -2157,43 +2233,39 @@ def SetUpConfigurations(target, target_dict): if not 'configurations' in target_dict: target_dict['configurations'] = {'Default': {}} if not 'default_configuration' in target_dict: - concrete = [i for i in target_dict['configurations'].iterkeys() - if not target_dict['configurations'][i].get('abstract')] + concrete = [i for (i, config) in target_dict['configurations'].iteritems() + if not config.get('abstract')] target_dict['default_configuration'] = sorted(concrete)[0] - for configuration in target_dict['configurations'].keys(): - old_configuration_dict = target_dict['configurations'][configuration] + merged_configurations = {} + configs = target_dict['configurations'] + for (configuration, old_configuration_dict) in configs.iteritems(): # Skip abstract configurations (saves work only). if old_configuration_dict.get('abstract'): continue # Configurations inherit (most) settings from the enclosing target scope. # Get the inheritance relationship right by making a copy of the target # dict. - new_configuration_dict = copy.deepcopy(target_dict) - - # Take out the bits that don't belong in a "configurations" section. - # Since configuration setup is done before conditional, exclude, and rules - # processing, be careful with handling of the suffix characters used in - # those phases. - delete_keys = [] - for key in new_configuration_dict: + new_configuration_dict = {} + for (key, target_val) in target_dict.iteritems(): key_ext = key[-1:] if key_ext in key_suffixes: key_base = key[:-1] else: key_base = key - if key_base in non_configuration_keys: - delete_keys.append(key) - - for key in delete_keys: - del new_configuration_dict[key] + if not key_base in non_configuration_keys: + new_configuration_dict[key] = gyp.simple_copy.deepcopy(target_val) # Merge in configuration (with all its parents first). MergeConfigWithInheritance(new_configuration_dict, build_file, target_dict, configuration, []) - # Put the new result back into the target dict as a configuration. - target_dict['configurations'][configuration] = new_configuration_dict + merged_configurations[configuration] = new_configuration_dict + + # Put the new configurations back into the target dict as a configuration. + for configuration in merged_configurations.keys(): + target_dict['configurations'][configuration] = ( + merged_configurations[configuration]) # Now drop all the abstract ones. for configuration in target_dict['configurations'].keys(): @@ -2264,9 +2336,9 @@ def ProcessListFiltersInDict(name, the_dict): if operation != '!' and operation != '/': continue - if not isinstance(value, list): - raise ValueError, name + ' key ' + key + ' must be list, not ' + \ - value.__class__.__name__ + if type(value) is not list: + raise ValueError(name + ' key ' + key + ' must be list, not ' + \ + value.__class__.__name__) list_key = key[:-1] if list_key not in the_dict: @@ -2276,12 +2348,12 @@ def ProcessListFiltersInDict(name, the_dict): del_lists.append(key) continue - if not isinstance(the_dict[list_key], list): + if type(the_dict[list_key]) is not list: value = the_dict[list_key] - raise ValueError, name + ' key ' + list_key + \ - ' must be list, not ' + \ - value.__class__.__name__ + ' when applying ' + \ - {'!': 'exclusion', '/': 'regex'}[operation] + raise ValueError(name + ' key ' + list_key + \ + ' must be list, not ' + \ + value.__class__.__name__ + ' when applying ' + \ + {'!': 'exclusion', '/': 'regex'}[operation]) if not list_key in lists: lists.append(list_key) @@ -2330,8 +2402,8 @@ def ProcessListFiltersInDict(name, the_dict): action_value = 1 else: # This is an action that doesn't make any sense. - raise ValueError, 'Unrecognized action ' + action + ' in ' + name + \ - ' key ' + regex_key + raise ValueError('Unrecognized action ' + action + ' in ' + name + \ + ' key ' + regex_key) for index in xrange(0, len(the_list)): list_item = the_list[index] @@ -2378,17 +2450,17 @@ def ProcessListFiltersInDict(name, the_dict): # Now recurse into subdicts and lists that may contain dicts. for key, value in the_dict.iteritems(): - if isinstance(value, dict): + if type(value) is dict: ProcessListFiltersInDict(key, value) - elif isinstance(value, list): + elif type(value) is list: ProcessListFiltersInList(key, value) def ProcessListFiltersInList(name, the_list): for item in the_list: - if isinstance(item, dict): + if type(item) is dict: ProcessListFiltersInDict(name, item) - elif isinstance(item, list): + elif type(item) is list: ProcessListFiltersInList(name, item) @@ -2416,9 +2488,11 @@ def ValidateTargetType(target, target_dict): target_type)) -def ValidateSourcesInTarget(target, target_dict, build_file): - # TODO: Check if MSVC allows this for loadable_module targets. - if target_dict.get('type', None) not in ('static_library', 'shared_library'): +def ValidateSourcesInTarget(target, target_dict, build_file, + duplicate_basename_check): + if not duplicate_basename_check: + return + if target_dict.get('type', None) != 'static_library': return sources = target_dict.get('sources', []) basenames = {} @@ -2438,8 +2512,8 @@ def ValidateSourcesInTarget(target, target_dict, build_file): if error: print('static library %s has several files with the same basename:\n' % - target + error + 'Some build systems, e.g. MSVC08, ' - 'cannot handle that.') + target + error + 'libtool on Mac cannot handle that. Use ' + '--no-duplicate-basename-check to disable this validation.') raise GypError('Duplicate basenames in sources section, see list above') @@ -2506,7 +2580,7 @@ def ValidateRunAsInTarget(target, target_dict, build_file): run_as = target_dict.get('run_as') if not run_as: return - if not isinstance(run_as, dict): + if type(run_as) is not dict: raise GypError("The 'run_as' in target %s from file %s should be a " "dictionary." % (target_name, build_file)) @@ -2515,17 +2589,17 @@ def ValidateRunAsInTarget(target, target_dict, build_file): raise GypError("The 'run_as' in target %s from file %s must have an " "'action' section." % (target_name, build_file)) - if not isinstance(action, list): + if type(action) is not list: raise GypError("The 'action' for 'run_as' in target %s from file %s " "must be a list." % (target_name, build_file)) working_directory = run_as.get('working_directory') - if working_directory and not isinstance(working_directory, str): + if working_directory and type(working_directory) is not str: raise GypError("The 'working_directory' for 'run_as' in target %s " "in file %s should be a string." % (target_name, build_file)) environment = run_as.get('environment') - if environment and not isinstance(environment, dict): + if environment and type(environment) is not dict: raise GypError("The 'environment' for 'run_as' in target %s " "in file %s should be a dictionary." % (target_name, build_file)) @@ -2555,17 +2629,17 @@ def TurnIntIntoStrInDict(the_dict): # Use items instead of iteritems because there's no need to try to look at # reinserted keys and their associated values. for k, v in the_dict.items(): - if isinstance(v, int): + if type(v) is int: v = str(v) the_dict[k] = v - elif isinstance(v, dict): + elif type(v) is dict: TurnIntIntoStrInDict(v) - elif isinstance(v, list): + elif type(v) is list: TurnIntIntoStrInList(v) - if isinstance(k, int): - the_dict[str(k)] = v + if type(k) is int: del the_dict[k] + the_dict[str(k)] = v def TurnIntIntoStrInList(the_list): @@ -2573,11 +2647,11 @@ def TurnIntIntoStrInList(the_list): """ for index in xrange(0, len(the_list)): item = the_list[index] - if isinstance(item, int): + if type(item) is int: the_list[index] = str(item) - elif isinstance(item, dict): + elif type(item) is dict: TurnIntIntoStrInDict(item) - elif isinstance(item, list): + elif type(item) is list: TurnIntIntoStrInList(item) @@ -2647,8 +2721,8 @@ def SetGeneratorGlobals(generator_input_info): # Set up path_sections and non_configuration_keys with the default data plus # the generator-specific data. global path_sections - path_sections = base_path_sections[:] - path_sections.extend(generator_input_info['path_sections']) + path_sections = set(base_path_sections) + path_sections.update(generator_input_info['path_sections']) global non_configuration_keys non_configuration_keys = base_non_configuration_keys[:] @@ -2663,7 +2737,7 @@ def SetGeneratorGlobals(generator_input_info): def Load(build_files, variables, includes, depth, generator_input_info, check, - circular_check, parallel, root_targets): + circular_check, duplicate_basename_check, parallel, root_targets): SetGeneratorGlobals(generator_input_info) # A generator can have other lists (in addition to sources) be processed # for rules. @@ -2677,15 +2751,14 @@ def Load(build_files, variables, includes, depth, generator_input_info, check, # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps # track of the keys corresponding to "target" files. data = {'target_build_files': set()} - aux_data = {} # Normalize paths everywhere. This is important because paths will be # used as keys to the data dict and for references between input files. build_files = set(map(os.path.normpath, build_files)) if parallel: - LoadTargetBuildFilesParallel(build_files, data, aux_data, - variables, includes, depth, check, - generator_input_info) + LoadTargetBuildFilesParallel(build_files, data, variables, includes, depth, + check, generator_input_info) else: + aux_data = {} for build_file in build_files: try: LoadTargetBuildFile(build_file, data, aux_data, @@ -2707,6 +2780,10 @@ def Load(build_files, variables, includes, depth, generator_input_info, check, # Expand dependencies specified as build_file:*. ExpandWildcardDependencies(targets, data) + # Remove all dependencies marked as 'link_dependency' from the targets of + # type 'none'. + RemoveLinkDependenciesFromNoneTargets(targets) + # Apply exclude (!) and regex (/) list filters only for dependency_sections. for target_name, target_dict in targets.iteritems(): tmp_dict = {} @@ -2792,10 +2869,8 @@ def Load(build_files, variables, includes, depth, generator_input_info, check, target_dict = targets[target] build_file = gyp.common.BuildFile(target) ValidateTargetType(target, target_dict) - # TODO(thakis): Get vpx_scale/arm/scalesystemdependent.c to be renamed to - # scalesystemdependent_arm_additions.c or similar. - if 'arm' not in variables.get('target_arch', ''): - ValidateSourcesInTarget(target, target_dict, build_file) + ValidateSourcesInTarget(target, target_dict, build_file, + duplicate_basename_check) ValidateRulesInTarget(target, target_dict, extra_sources_for_rules) ValidateRunAsInTarget(target, target_dict, build_file) ValidateActionsInTarget(target, target_dict, build_file) diff --git a/tools/gyp/pylib/gyp/input_test.py b/tools/gyp/pylib/gyp/input_test.py index cdbf6b2fad65..4234fbb8302e 100755 --- a/tools/gyp/pylib/gyp/input_test.py +++ b/tools/gyp/pylib/gyp/input_test.py @@ -44,16 +44,16 @@ def test_no_cycle_dag(self): def test_cycle_self_reference(self): self._create_dependency(self.nodes['a'], self.nodes['a']) - self.assertEquals([(self.nodes['a'], self.nodes['a'])], + self.assertEquals([[self.nodes['a'], self.nodes['a']]], self.nodes['a'].FindCycles()) def test_cycle_two_nodes(self): self._create_dependency(self.nodes['a'], self.nodes['b']) self._create_dependency(self.nodes['b'], self.nodes['a']) - self.assertEquals([(self.nodes['a'], self.nodes['b'], self.nodes['a'])], + self.assertEquals([[self.nodes['a'], self.nodes['b'], self.nodes['a']]], self.nodes['a'].FindCycles()) - self.assertEquals([(self.nodes['b'], self.nodes['a'], self.nodes['b'])], + self.assertEquals([[self.nodes['b'], self.nodes['a'], self.nodes['b']]], self.nodes['b'].FindCycles()) def test_two_cycles(self): @@ -65,9 +65,9 @@ def test_two_cycles(self): cycles = self.nodes['a'].FindCycles() self.assertTrue( - (self.nodes['a'], self.nodes['b'], self.nodes['a']) in cycles) + [self.nodes['a'], self.nodes['b'], self.nodes['a']] in cycles) self.assertTrue( - (self.nodes['b'], self.nodes['c'], self.nodes['b']) in cycles) + [self.nodes['b'], self.nodes['c'], self.nodes['b']] in cycles) self.assertEquals(2, len(cycles)) def test_big_cycle(self): @@ -77,12 +77,12 @@ def test_big_cycle(self): self._create_dependency(self.nodes['d'], self.nodes['e']) self._create_dependency(self.nodes['e'], self.nodes['a']) - self.assertEquals([(self.nodes['a'], + self.assertEquals([[self.nodes['a'], self.nodes['b'], self.nodes['c'], self.nodes['d'], self.nodes['e'], - self.nodes['a'])], + self.nodes['a']]], self.nodes['a'].FindCycles()) diff --git a/tools/gyp/pylib/gyp/mac_tool.py b/tools/gyp/pylib/gyp/mac_tool.py index ac19b6dc81fa..eeeaceb0c7aa 100755 --- a/tools/gyp/pylib/gyp/mac_tool.py +++ b/tools/gyp/pylib/gyp/mac_tool.py @@ -45,7 +45,7 @@ def _CommandifyName(self, name_string): """Transforms a tool name like copy-info-plist to CopyInfoPlist""" return name_string.title().replace('-', '') - def ExecCopyBundleResource(self, source, dest): + def ExecCopyBundleResource(self, source, dest, convert_to_binary): """Copies a resource file to the bundle/Resources directory, performing any necessary compilation on each resource.""" extension = os.path.splitext(source)[1].lower() @@ -62,7 +62,7 @@ def ExecCopyBundleResource(self, source, dest): elif extension == '.storyboard': return self._CopyXIBFile(source, dest) elif extension == '.strings': - self._CopyStringsFile(source, dest) + self._CopyStringsFile(source, dest, convert_to_binary) else: shutil.copy(source, dest) @@ -92,7 +92,11 @@ def _CopyXIBFile(self, source, dest): sys.stdout.write(line) return ibtoolout.returncode - def _CopyStringsFile(self, source, dest): + def _ConvertToBinary(self, dest): + subprocess.check_call([ + 'xcrun', 'plutil', '-convert', 'binary1', '-o', dest, dest]) + + def _CopyStringsFile(self, source, dest, convert_to_binary): """Copies a .strings file using iconv to reconvert the input into UTF-16.""" input_code = self._DetectInputEncoding(source) or "UTF-8" @@ -112,6 +116,9 @@ def _CopyStringsFile(self, source, dest): fp.write(s.decode(input_code).encode('UTF-16')) fp.close() + if convert_to_binary == 'True': + self._ConvertToBinary(dest) + def _DetectInputEncoding(self, file_name): """Reads the first few bytes from file_name and tries to guess the text encoding. Returns None as a guess if it can't detect it.""" @@ -131,7 +138,7 @@ def _DetectInputEncoding(self, file_name): else: return None - def ExecCopyInfoPlist(self, source, dest, *keys): + def ExecCopyInfoPlist(self, source, dest, convert_to_binary, *keys): """Copies the |source| Info.plist to the destination directory |dest|.""" # Read the source Info.plist into memory. fd = open(source, 'r') @@ -146,7 +153,7 @@ def ExecCopyInfoPlist(self, source, dest, *keys): # Go through all the environment variables and replace them as variables in # the file. - IDENT_RE = re.compile('[/\s]') + IDENT_RE = re.compile(r'[/\s]') for key in os.environ: if key.startswith('_'): continue @@ -185,6 +192,9 @@ def ExecCopyInfoPlist(self, source, dest, *keys): # "compiled". self._WritePkgInfo(dest) + if convert_to_binary == 'True': + self._ConvertToBinary(dest) + def _WritePkgInfo(self, info_plist): """This writes the PkgInfo file from the data stored in Info.plist.""" plist = plistlib.readPlist(info_plist) @@ -219,11 +229,28 @@ def ExecFilterLibtool(self, *cmd_list): """Calls libtool and filters out '/path/to/libtool: file: foo.o has no symbols'.""" libtool_re = re.compile(r'^.*libtool: file: .* has no symbols$') - libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE) + libtool_re5 = re.compile( + r'^.*libtool: warning for library: ' + + r'.* the table of contents is empty ' + + r'\(no object file members in the library define global symbols\)$') + env = os.environ.copy() + # Ref: + # http://www.opensource.apple.com/source/cctools/cctools-809/misc/libtool.c + # The problem with this flag is that it resets the file mtime on the file to + # epoch=0, e.g. 1970-1-1 or 1969-12-31 depending on timezone. + env['ZERO_AR_DATE'] = '1' + libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env) _, err = libtoolout.communicate() for line in err.splitlines(): - if not libtool_re.match(line): + if not libtool_re.match(line) and not libtool_re5.match(line): print >>sys.stderr, line + # Unconditionally touch the output .a file on the command line if present + # and the command succeeded. A bit hacky. + if not libtoolout.returncode: + for i in range(len(cmd_list) - 1): + if cmd_list[i] == "-o" and cmd_list[i+1].endswith('.a'): + os.utime(cmd_list[i+1], None) + break return libtoolout.returncode def ExecPackageFramework(self, framework, version): @@ -262,6 +289,66 @@ def _Relink(self, dest, link): os.remove(link) os.symlink(dest, link) + def ExecCompileXcassets(self, keys, *inputs): + """Compiles multiple .xcassets files into a single .car file. + + This invokes 'actool' to compile all the inputs .xcassets files. The + |keys| arguments is a json-encoded dictionary of extra arguments to + pass to 'actool' when the asset catalogs contains an application icon + or a launch image. + + Note that 'actool' does not create the Assets.car file if the asset + catalogs does not contains imageset. + """ + command_line = [ + 'xcrun', 'actool', '--output-format', 'human-readable-text', + '--compress-pngs', '--notices', '--warnings', '--errors', + ] + is_iphone_target = 'IPHONEOS_DEPLOYMENT_TARGET' in os.environ + if is_iphone_target: + platform = os.environ['CONFIGURATION'].split('-')[-1] + if platform not in ('iphoneos', 'iphonesimulator'): + platform = 'iphonesimulator' + command_line.extend([ + '--platform', platform, '--target-device', 'iphone', + '--target-device', 'ipad', '--minimum-deployment-target', + os.environ['IPHONEOS_DEPLOYMENT_TARGET'], '--compile', + os.path.abspath(os.environ['CONTENTS_FOLDER_PATH']), + ]) + else: + command_line.extend([ + '--platform', 'macosx', '--target-device', 'mac', + '--minimum-deployment-target', os.environ['MACOSX_DEPLOYMENT_TARGET'], + '--compile', + os.path.abspath(os.environ['UNLOCALIZED_RESOURCES_FOLDER_PATH']), + ]) + if keys: + keys = json.loads(keys) + for key, value in keys.iteritems(): + arg_name = '--' + key + if isinstance(value, bool): + if value: + command_line.append(arg_name) + elif isinstance(value, list): + for v in value: + command_line.append(arg_name) + command_line.append(str(v)) + else: + command_line.append(arg_name) + command_line.append(str(value)) + # Note: actool crashes if inputs path are relative, so use os.path.abspath + # to get absolute path name for inputs. + command_line.extend(map(os.path.abspath, inputs)) + subprocess.check_call(command_line) + + def ExecMergeInfoPlist(self, output, *inputs): + """Merge multiple .plist files into a single .plist file.""" + merged_plist = {} + for path in inputs: + plist = self._LoadPlistMaybeBinary(path) + self._MergePlist(merged_plist, plist) + plistlib.writePlist(merged_plist, output) + def ExecCodeSignBundle(self, key, resource_rules, entitlements, provisioning): """Code sign a bundle. @@ -398,6 +485,19 @@ def _LoadProvisioningProfile(self, profile_path): 'security', 'cms', '-D', '-i', profile_path, '-o', temp.name]) return self._LoadPlistMaybeBinary(temp.name) + def _MergePlist(self, merged_plist, plist): + """Merge |plist| into |merged_plist|.""" + for key, value in plist.iteritems(): + if isinstance(value, dict): + merged_value = merged_plist.get(key, {}) + if isinstance(merged_value, dict): + self._MergePlist(merged_value, value) + merged_plist[key] = merged_value + else: + merged_plist[key] = value + else: + merged_plist[key] = value + def _LoadPlistMaybeBinary(self, plist_path): """Loads into a memory a plist possibly encoded in binary format. @@ -503,8 +603,7 @@ def _ExpandVariables(self, data, substitutions): if isinstance(data, list): return [self._ExpandVariables(v, substitutions) for v in data] if isinstance(data, dict): - return dict((k, self._ExpandVariables(data[k], - substitutions)) for k in data) + return {k: self._ExpandVariables(data[k], substitutions) for k in data} return data if __name__ == '__main__': diff --git a/tools/gyp/pylib/gyp/msvs_emulation.py b/tools/gyp/pylib/gyp/msvs_emulation.py index 6428fced0055..ce5c46ea5b3d 100644 --- a/tools/gyp/pylib/gyp/msvs_emulation.py +++ b/tools/gyp/pylib/gyp/msvs_emulation.py @@ -12,10 +12,14 @@ import subprocess import sys +from gyp.common import OrderedSet +import gyp.MSVSUtil import gyp.MSVSVersion + windows_quoter_regex = re.compile(r'(\\*)"') + def QuoteForRspFile(arg): """Quote a command line argument so that it appears as one argument when processed via cmd.exe and parsed by CommandLineToArgvW (as is typical for @@ -131,6 +135,54 @@ def _FindDirectXInstallation(): return dxsdk_dir +def GetGlobalVSMacroEnv(vs_version): + """Get a dict of variables mapping internal VS macro names to their gyp + equivalents. Returns all variables that are independent of the target.""" + env = {} + # '$(VSInstallDir)' and '$(VCInstallDir)' are available when and only when + # Visual Studio is actually installed. + if vs_version.Path(): + env['$(VSInstallDir)'] = vs_version.Path() + env['$(VCInstallDir)'] = os.path.join(vs_version.Path(), 'VC') + '\\' + # Chromium uses DXSDK_DIR in include/lib paths, but it may or may not be + # set. This happens when the SDK is sync'd via src-internal, rather than + # by typical end-user installation of the SDK. If it's not set, we don't + # want to leave the unexpanded variable in the path, so simply strip it. + dxsdk_dir = _FindDirectXInstallation() + env['$(DXSDK_DIR)'] = dxsdk_dir if dxsdk_dir else '' + # Try to find an installation location for the Windows DDK by checking + # the WDK_DIR environment variable, may be None. + env['$(WDK_DIR)'] = os.environ.get('WDK_DIR', '') + return env + +def ExtractSharedMSVSSystemIncludes(configs, generator_flags): + """Finds msvs_system_include_dirs that are common to all targets, removes + them from all targets, and returns an OrderedSet containing them.""" + all_system_includes = OrderedSet( + configs[0].get('msvs_system_include_dirs', [])) + for config in configs[1:]: + system_includes = config.get('msvs_system_include_dirs', []) + all_system_includes = all_system_includes & OrderedSet(system_includes) + if not all_system_includes: + return None + # Expand macros in all_system_includes. + env = GetGlobalVSMacroEnv(GetVSVersion(generator_flags)) + expanded_system_includes = OrderedSet([ExpandMacros(include, env) + for include in all_system_includes]) + if any(['$' in include for include in expanded_system_includes]): + # Some path relies on target-specific variables, bail. + return None + + # Remove system includes shared by all targets from the targets. + for config in configs: + includes = config.get('msvs_system_include_dirs', []) + if includes: # Don't insert a msvs_system_include_dirs key if not needed. + # This must check the unexpanded includes list: + new_includes = [i for i in includes if i not in all_system_includes] + config['msvs_system_include_dirs'] = new_includes + return expanded_system_includes + + class MsvsSettings(object): """A class that understands the gyp 'msvs_...' values (especially the msvs_settings field). They largely correpond to the VS2008 IDE DOM. This @@ -139,11 +191,6 @@ class helps map those settings to command line options.""" def __init__(self, spec, generator_flags): self.spec = spec self.vs_version = GetVSVersion(generator_flags) - self.dxsdk_dir = _FindDirectXInstallation() - - # Try to find an installation location for the Windows DDK by checking - # the WDK_DIR environment variable, may be None. - self.wdk_dir = os.environ.get('WDK_DIR') supported_fields = [ ('msvs_configuration_attributes', dict), @@ -163,6 +210,30 @@ def __init__(self, spec, generator_flags): self.msvs_cygwin_dirs = spec.get('msvs_cygwin_dirs', ['.']) + unsupported_fields = [ + 'msvs_prebuild', + 'msvs_postbuild', + ] + unsupported = [] + for field in unsupported_fields: + for config in configs.values(): + if field in config: + unsupported += ["%s not supported (target %s)." % + (field, spec['target_name'])] + if unsupported: + raise Exception('\n'.join(unsupported)) + + def GetExtension(self): + """Returns the extension for the target, with no leading dot. + + Uses 'product_extension' if specified, otherwise uses MSVS defaults based on + the target type. + """ + ext = self.spec.get('product_extension', None) + if ext: + return ext + return gyp.MSVSUtil.TARGET_TYPE_EXT.get(self.spec['type'], '') + def GetVSMacroEnv(self, base_to_build=None, config=None): """Get a dict of variables mapping internal VS macro names to their gyp equivalents.""" @@ -170,29 +241,24 @@ def GetVSMacroEnv(self, base_to_build=None, config=None): target_name = self.spec.get('product_prefix', '') + \ self.spec.get('product_name', self.spec['target_name']) target_dir = base_to_build + '\\' if base_to_build else '' + target_ext = '.' + self.GetExtension() + target_file_name = target_name + target_ext + replacements = { - '$(OutDir)\\': target_dir, - '$(TargetDir)\\': target_dir, - '$(IntDir)': '$!INTERMEDIATE_DIR', - '$(InputPath)': '${source}', '$(InputName)': '${root}', - '$(ProjectName)': self.spec['target_name'], - '$(TargetName)': target_name, + '$(InputPath)': '${source}', + '$(IntDir)': '$!INTERMEDIATE_DIR', + '$(OutDir)\\': target_dir, '$(PlatformName)': target_platform, '$(ProjectDir)\\': '', + '$(ProjectName)': self.spec['target_name'], + '$(TargetDir)\\': target_dir, + '$(TargetExt)': target_ext, + '$(TargetFileName)': target_file_name, + '$(TargetName)': target_name, + '$(TargetPath)': os.path.join(target_dir, target_file_name), } - # '$(VSInstallDir)' and '$(VCInstallDir)' are available when and only when - # Visual Studio is actually installed. - if self.vs_version.Path(): - replacements['$(VSInstallDir)'] = self.vs_version.Path() - replacements['$(VCInstallDir)'] = os.path.join(self.vs_version.Path(), - 'VC') + '\\' - # Chromium uses DXSDK_DIR in include/lib paths, but it may or may not be - # set. This happens when the SDK is sync'd via src-internal, rather than - # by typical end-user installation of the SDK. If it's not set, we don't - # want to leave the unexpanded variable in the path, so simply strip it. - replacements['$(DXSDK_DIR)'] = self.dxsdk_dir if self.dxsdk_dir else '' - replacements['$(WDK_DIR)'] = self.wdk_dir if self.wdk_dir else '' + replacements.update(GetGlobalVSMacroEnv(self.vs_version)) return replacements def ConvertVSMacros(self, s, base_to_build=None, config=None): @@ -272,6 +338,15 @@ def AdjustIncludeDirs(self, include_dirs, config): ('VCCLCompilerTool', 'AdditionalIncludeDirectories'), config, default=[])) return [self.ConvertVSMacros(p, config=config) for p in includes] + def AdjustMidlIncludeDirs(self, midl_include_dirs, config): + """Updates midl_include_dirs to expand VS specific paths, and adds the + system include dirs used for platform SDK and similar.""" + config = self._TargetConfig(config) + includes = midl_include_dirs + self.msvs_system_include_dirs[config] + includes.extend(self._Setting( + ('VCMIDLTool', 'AdditionalIncludeDirectories'), config, default=[])) + return [self.ConvertVSMacros(p, config=config) for p in includes] + def GetComputedDefines(self, config): """Returns the set of defines that are injected to the defines list based on other VS settings.""" @@ -324,7 +399,7 @@ def GetPDBName(self, config, expand_special, default): output_file = self._Setting(('VCLinkerTool', 'ProgramDatabaseFile'), config) generate_debug_info = self._Setting( ('VCLinkerTool', 'GenerateDebugInformation'), config) - if generate_debug_info: + if generate_debug_info == 'true': if output_file: return expand_special(self.ConvertVSMacros(output_file, config=config)) else: @@ -332,6 +407,22 @@ def GetPDBName(self, config, expand_special, default): else: return None + def GetNoImportLibrary(self, config): + """If NoImportLibrary: true, ninja will not expect the output to include + an import library.""" + config = self._TargetConfig(config) + noimplib = self._Setting(('NoImportLibrary',), config) + return noimplib == 'true' + + def GetAsmflags(self, config): + """Returns the flags that need to be added to ml invocations.""" + config = self._TargetConfig(config) + asmflags = [] + safeseh = self._Setting(('MASM', 'UseSafeExceptionHandlers'), config) + if safeseh == 'true': + asmflags.append('/safeseh') + return asmflags + def GetCflags(self, config): """Returns the flags that need to be added to .c and .cc compilations.""" config = self._TargetConfig(config) @@ -348,9 +439,14 @@ def GetCflags(self, config): cl('OmitFramePointers', map={'false': '-', 'true': ''}, prefix='/Oy') cl('EnableIntrinsicFunctions', map={'false': '-', 'true': ''}, prefix='/Oi') cl('FavorSizeOrSpeed', map={'1': 't', '2': 's'}, prefix='/O') + cl('FloatingPointModel', + map={'0': 'precise', '1': 'strict', '2': 'fast'}, prefix='/fp:', + default='0') cl('WholeProgramOptimization', map={'true': '/GL'}) cl('WarningLevel', prefix='/W') cl('WarnAsError', map={'true': '/WX'}) + cl('CallingConvention', + map={'0': 'd', '1': 'r', '2': 'z', '3': 'v'}, prefix='/G') cl('DebugInformationFormat', map={'1': '7', '3': 'i', '4': 'I'}, prefix='/Z') cl('RuntimeTypeInfo', map={'true': '/GR', 'false': '/GR-'}) @@ -366,21 +462,18 @@ def GetCflags(self, config): map={'false': '-', 'true': ''}, prefix='/Zc:wchar_t') cl('EnablePREfast', map={'true': '/analyze'}) cl('AdditionalOptions', prefix='') + cl('EnableEnhancedInstructionSet', + map={'1': 'SSE', '2': 'SSE2', '3': 'AVX', '4': 'IA32', '5': 'AVX2'}, + prefix='/arch:') cflags.extend(['/FI' + f for f in self._Setting( ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])]) - if self.vs_version.short_name in ('2013', '2013e'): + if self.vs_version.short_name in ('2013', '2013e', '2015'): # New flag required in 2013 to maintain previous PDB behavior. cflags.append('/FS') # ninja handles parallelism by itself, don't have the compiler do it too. cflags = filter(lambda x: not x.startswith('/MP'), cflags) return cflags - def GetPrecompiledHeader(self, config, gyp_to_build_path): - """Returns an object that handles the generation of precompiled header - build steps.""" - config = self._TargetConfig(config) - return _PchHelper(self, config, gyp_to_build_path) - def _GetPchFlags(self, config, extension): """Get the flags to be added to the cflags for precompiled header support. """ @@ -425,7 +518,8 @@ def GetLibFlags(self, config, gyp_to_build_path): libflags.extend(self._GetAdditionalLibraryDirectories( 'VCLibrarianTool', config, gyp_to_build_path)) lib('LinkTimeCodeGeneration', map={'true': '/LTCG'}) - lib('TargetMachine', map={'1': 'X86', '17': 'X64'}, prefix='/MACHINE:') + lib('TargetMachine', map={'1': 'X86', '17': 'X64', '3': 'ARM'}, + prefix='/MACHINE:') lib('AdditionalOptions') return libflags @@ -468,7 +562,8 @@ def GetLdflags(self, config, gyp_to_build_path, expand_special, 'VCLinkerTool', append=ldflags) self._GetDefFileAsLdflags(ldflags, gyp_to_build_path) ld('GenerateDebugInformation', map={'true': '/DEBUG'}) - ld('TargetMachine', map={'1': 'X86', '17': 'X64'}, prefix='/MACHINE:') + ld('TargetMachine', map={'1': 'X86', '17': 'X64', '3': 'ARM'}, + prefix='/MACHINE:') ldflags.extend(self._GetAdditionalLibraryDirectories( 'VCLinkerTool', config, gyp_to_build_path)) ld('DelayLoadDLLs', prefix='/DELAYLOAD:') @@ -522,6 +617,14 @@ def GetLdflags(self, config, gyp_to_build_path, expand_special, # TODO(scottmg): This should sort of be somewhere else (not really a flag). ld('AdditionalDependencies', prefix='') + if self.GetArch(config) == 'x86': + safeseh_default = 'true' + else: + safeseh_default = None + ld('ImageHasSafeExceptionHandlers', + map={'false': ':NO', 'true': ''}, prefix='/SAFESEH', + default=safeseh_default) + # If the base address is not specifically controlled, DYNAMICBASE should # be on by default. base_flags = filter(lambda x: 'DYNAMICBASE' in x or x == '/FIXED', @@ -708,10 +811,16 @@ def _HasExplicitRuleForExtension(self, spec, extension): return True return False - def HasExplicitIdlRules(self, spec): - """Determine if there's an explicit rule for idl files. When there isn't we - need to generate implicit rules to build MIDL .idl files.""" - return self._HasExplicitRuleForExtension(spec, 'idl') + def _HasExplicitIdlActions(self, spec): + """Determine if an action should not run midl for .idl files.""" + return any([action.get('explicit_idl_action', 0) + for action in spec.get('actions', [])]) + + def HasExplicitIdlRulesOrActions(self, spec): + """Determine if there's an explicit rule or action for idl files. When + there isn't we need to generate implicit rules to build MIDL .idl files.""" + return (self._HasExplicitRuleForExtension(spec, 'idl') or + self._HasExplicitIdlActions(spec)) def HasExplicitAsmRules(self, spec): """Determine if there's an explicit rule for asm files. When there isn't we @@ -774,7 +883,7 @@ def _PchHeader(self): def GetObjDependencies(self, sources, objs, arch): """Given a list of sources files and the corresponding object files, returns a list of the pch files that should be depended upon. The - additional wrapping in the return value is for interface compatability + additional wrapping in the return value is for interface compatibility with make.py on Mac, and xcode_emulation.py.""" assert arch is None if not self._PchHeader(): @@ -810,7 +919,8 @@ def GetVSVersion(generator_flags): global vs_version if not vs_version: vs_version = gyp.MSVSVersion.SelectVisualStudioVersion( - generator_flags.get('msvs_version', 'auto')) + generator_flags.get('msvs_version', 'auto'), + allow_fallback=False) return vs_version def _GetVsvarsSetupArgs(generator_flags, arch): @@ -878,7 +988,8 @@ def _ExtractCLPath(output_of_where): if line.startswith('LOC:'): return line[len('LOC:'):].strip() -def GenerateEnvironmentFiles(toplevel_build_dir, generator_flags, open_out): +def GenerateEnvironmentFiles(toplevel_build_dir, generator_flags, + system_includes, open_out): """It's not sufficient to have the absolute path to the compiler, linker, etc. on Windows, as those tools rely on .dlls being in the PATH. We also need to support both x86 and x64 compilers within the same build (to support @@ -909,6 +1020,13 @@ def GenerateEnvironmentFiles(toplevel_build_dir, generator_flags, open_out): args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) variables, _ = popen.communicate() env = _ExtractImportantEnvironment(variables) + + # Inject system includes from gyp files into INCLUDE. + if system_includes: + system_includes = system_includes | OrderedSet( + env.get('INCLUDE', '').split(';')) + env['INCLUDE'] = ';'.join(system_includes) + env_block = _FormatAsEnvironmentBlock(env) f = open_out(os.path.join(toplevel_build_dir, 'environment.' + arch), 'wb') f.write(env_block) diff --git a/tools/gyp/pylib/gyp/simple_copy.py b/tools/gyp/pylib/gyp/simple_copy.py new file mode 100644 index 000000000000..74c98c5a7959 --- /dev/null +++ b/tools/gyp/pylib/gyp/simple_copy.py @@ -0,0 +1,46 @@ +# Copyright 2014 Google Inc. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A clone of the default copy.deepcopy that doesn't handle cyclic +structures or complex types except for dicts and lists. This is +because gyp copies so large structure that small copy overhead ends up +taking seconds in a project the size of Chromium.""" + +class Error(Exception): + pass + +__all__ = ["Error", "deepcopy"] + +def deepcopy(x): + """Deep copy operation on gyp objects such as strings, ints, dicts + and lists. More than twice as fast as copy.deepcopy but much less + generic.""" + + try: + return _deepcopy_dispatch[type(x)](x) + except KeyError: + raise Error('Unsupported type %s for deepcopy. Use copy.deepcopy ' + + 'or expand simple_copy support.' % type(x)) + +_deepcopy_dispatch = d = {} + +def _deepcopy_atomic(x): + return x + +for x in (type(None), int, long, float, + bool, str, unicode, type): + d[x] = _deepcopy_atomic + +def _deepcopy_list(x): + return [deepcopy(a) for a in x] +d[list] = _deepcopy_list + +def _deepcopy_dict(x): + y = {} + for key, value in x.iteritems(): + y[deepcopy(key)] = deepcopy(value) + return y +d[dict] = _deepcopy_dict + +del d diff --git a/tools/gyp/pylib/gyp/win_tool.py b/tools/gyp/pylib/gyp/win_tool.py index e9d7df0bb771..417e465f7853 100755 --- a/tools/gyp/pylib/gyp/win_tool.py +++ b/tools/gyp/pylib/gyp/win_tool.py @@ -13,6 +13,7 @@ import re import shutil import subprocess +import stat import string import sys @@ -48,7 +49,8 @@ def _UseSeparateMspdbsrv(self, env, args): for arg in args: m = _LINK_EXE_OUT_ARG.match(arg) if m: - endpoint_name = '%s_%d' % (m.group('out'), os.getpid()) + endpoint_name = re.sub(r'\W+', '', + '%s_%d' % (m.group('out'), os.getpid())) break if endpoint_name is None: @@ -88,9 +90,19 @@ def ExecRecursiveMirror(self, source, dest): """Emulation of rm -rf out && cp -af in out.""" if os.path.exists(dest): if os.path.isdir(dest): - shutil.rmtree(dest) + def _on_error(fn, path, excinfo): + # The operation failed, possibly because the file is set to + # read-only. If that's why, make it writable and try the op again. + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IWRITE) + fn(path) + shutil.rmtree(dest, onerror=_on_error) else: + if not os.access(dest, os.W_OK): + # Attempt to make the file writable before deleting it. + os.chmod(dest, stat.S_IWRITE) os.unlink(dest) + if os.path.isdir(source): shutil.copytree(source, dest) else: @@ -104,7 +116,7 @@ def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): env = self._GetEnv(arch) if use_separate_mspdbsrv == 'True': self._UseSeparateMspdbsrv(env, args) - link = subprocess.Popen(args, + link = subprocess.Popen([args[0].replace('/', '\\')] + list(args[1:]), shell=True, env=env, stdout=subprocess.PIPE, @@ -236,19 +248,17 @@ def ExecMidlWrapper(self, arch, outdir, tlb, h, dlldata, iid, proxy, idl, # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl # objidl.idl lines = out.splitlines() - prefix = 'Processing ' - processing = set(os.path.basename(x) for x in lines if x.startswith(prefix)) + prefixes = ('Processing ', '64 bit Processing ') + processing = set(os.path.basename(x) + for x in lines if x.startswith(prefixes)) for line in lines: - if not line.startswith(prefix) and line not in processing: + if not line.startswith(prefixes) and line not in processing: print line return popen.returncode def ExecAsmWrapper(self, arch, *args): """Filter logo banner from invocations of asm.exe.""" env = self._GetEnv(arch) - # MSVS doesn't assemble x64 asm files. - if arch == 'environment.x64': - return 0 popen = subprocess.Popen(args, shell=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = popen.communicate() @@ -287,5 +297,16 @@ def ExecActionWrapper(self, arch, rspfile, *dir): dir = dir[0] if dir else None return subprocess.call(args, shell=True, env=env, cwd=dir) + def ExecClCompile(self, project_dir, selected_files): + """Executed by msvs-ninja projects when the 'ClCompile' target is used to + build selected C/C++ files.""" + project_dir = os.path.relpath(project_dir, BASE_DIR) + selected_files = selected_files.split(';') + ninja_targets = [os.path.join(project_dir, filename) + '^^' + for filename in selected_files] + cmd = ['ninja.exe'] + cmd.extend(ninja_targets) + return subprocess.call(cmd, shell=True, cwd=BASE_DIR) + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/tools/gyp/pylib/gyp/xcode_emulation.py b/tools/gyp/pylib/gyp/xcode_emulation.py index 30f27d5832d9..47f9d520e10e 100644 --- a/tools/gyp/pylib/gyp/xcode_emulation.py +++ b/tools/gyp/pylib/gyp/xcode_emulation.py @@ -18,6 +18,129 @@ import tempfile from gyp.common import GypError +# Populated lazily by XcodeVersion, for efficiency, and to fix an issue when +# "xcodebuild" is called too quickly (it has been found to return incorrect +# version number). +XCODE_VERSION_CACHE = None + +# Populated lazily by GetXcodeArchsDefault, to an |XcodeArchsDefault| instance +# corresponding to the installed version of Xcode. +XCODE_ARCHS_DEFAULT_CACHE = None + + +def XcodeArchsVariableMapping(archs, archs_including_64_bit=None): + """Constructs a dictionary with expansion for $(ARCHS_STANDARD) variable, + and optionally for $(ARCHS_STANDARD_INCLUDING_64_BIT).""" + mapping = {'$(ARCHS_STANDARD)': archs} + if archs_including_64_bit: + mapping['$(ARCHS_STANDARD_INCLUDING_64_BIT)'] = archs_including_64_bit + return mapping + +class XcodeArchsDefault(object): + """A class to resolve ARCHS variable from xcode_settings, resolving Xcode + macros and implementing filtering by VALID_ARCHS. The expansion of macros + depends on the SDKROOT used ("macosx", "iphoneos", "iphonesimulator") and + on the version of Xcode. + """ + + # Match variable like $(ARCHS_STANDARD). + variable_pattern = re.compile(r'\$\([a-zA-Z_][a-zA-Z0-9_]*\)$') + + def __init__(self, default, mac, iphonesimulator, iphoneos): + self._default = (default,) + self._archs = {'mac': mac, 'ios': iphoneos, 'iossim': iphonesimulator} + + def _VariableMapping(self, sdkroot): + """Returns the dictionary of variable mapping depending on the SDKROOT.""" + sdkroot = sdkroot.lower() + if 'iphoneos' in sdkroot: + return self._archs['ios'] + elif 'iphonesimulator' in sdkroot: + return self._archs['iossim'] + else: + return self._archs['mac'] + + def _ExpandArchs(self, archs, sdkroot): + """Expands variables references in ARCHS, and remove duplicates.""" + variable_mapping = self._VariableMapping(sdkroot) + expanded_archs = [] + for arch in archs: + if self.variable_pattern.match(arch): + variable = arch + try: + variable_expansion = variable_mapping[variable] + for arch in variable_expansion: + if arch not in expanded_archs: + expanded_archs.append(arch) + except KeyError as e: + print 'Warning: Ignoring unsupported variable "%s".' % variable + elif arch not in expanded_archs: + expanded_archs.append(arch) + return expanded_archs + + def ActiveArchs(self, archs, valid_archs, sdkroot): + """Expands variables references in ARCHS, and filter by VALID_ARCHS if it + is defined (if not set, Xcode accept any value in ARCHS, otherwise, only + values present in VALID_ARCHS are kept).""" + expanded_archs = self._ExpandArchs(archs or self._default, sdkroot or '') + if valid_archs: + filtered_archs = [] + for arch in expanded_archs: + if arch in valid_archs: + filtered_archs.append(arch) + expanded_archs = filtered_archs + return expanded_archs + + +def GetXcodeArchsDefault(): + """Returns the |XcodeArchsDefault| object to use to expand ARCHS for the + installed version of Xcode. The default values used by Xcode for ARCHS + and the expansion of the variables depends on the version of Xcode used. + + For all version anterior to Xcode 5.0 or posterior to Xcode 5.1 included + uses $(ARCHS_STANDARD) if ARCHS is unset, while Xcode 5.0 to 5.0.2 uses + $(ARCHS_STANDARD_INCLUDING_64_BIT). This variable was added to Xcode 5.0 + and deprecated with Xcode 5.1. + + For "macosx" SDKROOT, all version starting with Xcode 5.0 includes 64-bit + architecture as part of $(ARCHS_STANDARD) and default to only building it. + + For "iphoneos" and "iphonesimulator" SDKROOT, 64-bit architectures are part + of $(ARCHS_STANDARD_INCLUDING_64_BIT) from Xcode 5.0. From Xcode 5.1, they + are also part of $(ARCHS_STANDARD). + + All thoses rules are coded in the construction of the |XcodeArchsDefault| + object to use depending on the version of Xcode detected. The object is + for performance reason.""" + global XCODE_ARCHS_DEFAULT_CACHE + if XCODE_ARCHS_DEFAULT_CACHE: + return XCODE_ARCHS_DEFAULT_CACHE + xcode_version, _ = XcodeVersion() + if xcode_version < '0500': + XCODE_ARCHS_DEFAULT_CACHE = XcodeArchsDefault( + '$(ARCHS_STANDARD)', + XcodeArchsVariableMapping(['i386']), + XcodeArchsVariableMapping(['i386']), + XcodeArchsVariableMapping(['armv7'])) + elif xcode_version < '0510': + XCODE_ARCHS_DEFAULT_CACHE = XcodeArchsDefault( + '$(ARCHS_STANDARD_INCLUDING_64_BIT)', + XcodeArchsVariableMapping(['x86_64'], ['x86_64']), + XcodeArchsVariableMapping(['i386'], ['i386', 'x86_64']), + XcodeArchsVariableMapping( + ['armv7', 'armv7s'], + ['armv7', 'armv7s', 'arm64'])) + else: + XCODE_ARCHS_DEFAULT_CACHE = XcodeArchsDefault( + '$(ARCHS_STANDARD)', + XcodeArchsVariableMapping(['x86_64'], ['x86_64']), + XcodeArchsVariableMapping(['i386', 'x86_64'], ['i386', 'x86_64']), + XcodeArchsVariableMapping( + ['armv7', 'armv7s', 'arm64'], + ['armv7', 'armv7s', 'arm64'])) + return XCODE_ARCHS_DEFAULT_CACHE + + class XcodeSettings(object): """A class that understands the gyp 'xcode_settings' object.""" @@ -34,10 +157,6 @@ class XcodeSettings(object): # cached at class-level for efficiency. _codesigning_key_cache = {} - # Populated lazily by _XcodeVersion. Shared by all XcodeSettings, so cached - # at class-level for efficiency. - _xcode_version_cache = () - def __init__(self, spec): self.spec = spec @@ -96,9 +215,24 @@ def _WarnUnimplemented(self, test_key): if test_key in self._Settings(): print 'Warning: Ignoring not yet implemented key "%s".' % test_key + def IsBinaryOutputFormat(self, configname): + default = "binary" if self.isIOS else "xml" + format = self.xcode_settings[configname].get('INFOPLIST_OUTPUT_FORMAT', + default) + return format == "binary" + def _IsBundle(self): return int(self.spec.get('mac_bundle', 0)) != 0 + def _IsIosAppExtension(self): + return int(self.spec.get('ios_app_extension', 0)) != 0 + + def _IsIosWatchKitExtension(self): + return int(self.spec.get('ios_watchkit_extension', 0)) != 0 + + def _IsIosWatchApp(self): + return int(self.spec.get('ios_watch_app', 0)) != 0 + def GetFrameworkVersion(self): """Returns the framework version of the current target. Only valid for bundles.""" @@ -118,7 +252,10 @@ def GetWrapperExtension(self): 'WRAPPER_EXTENSION', default=default_wrapper_extension) return '.' + self.spec.get('product_extension', wrapper_extension) elif self.spec['type'] == 'executable': - return '.' + self.spec.get('product_extension', 'app') + if self._IsIosAppExtension() or self._IsIosWatchKitExtension(): + return '.' + self.spec.get('product_extension', 'appex') + else: + return '.' + self.spec.get('product_extension', 'app') else: assert False, "Don't know extension for '%s', target '%s'" % ( self.spec['type'], self.spec['target_name']) @@ -173,6 +310,18 @@ def GetBundlePlistPath(self): def GetProductType(self): """Returns the PRODUCT_TYPE of this target.""" + if self._IsIosAppExtension(): + assert self._IsBundle(), ('ios_app_extension flag requires mac_bundle ' + '(target %s)' % self.spec['target_name']) + return 'com.apple.product-type.app-extension' + if self._IsIosWatchKitExtension(): + assert self._IsBundle(), ('ios_watchkit_extension flag requires ' + 'mac_bundle (target %s)' % self.spec['target_name']) + return 'com.apple.product-type.watchkit-extension' + if self._IsIosWatchApp(): + assert self._IsBundle(), ('ios_watch_app flag requires mac_bundle ' + '(target %s)' % self.spec['target_name']) + return 'com.apple.product-type.application.watchapp' if self._IsBundle(): return { 'executable': 'com.apple.product-type.application', @@ -267,17 +416,12 @@ def GetExecutablePath(self): def GetActiveArchs(self, configname): """Returns the architectures this target should be built for.""" - # TODO: Look at VALID_ARCHS, ONLY_ACTIVE_ARCH; possibly set - # CURRENT_ARCH / NATIVE_ARCH env vars? - return self.xcode_settings[configname].get('ARCHS', [self._DefaultArch()]) - - def _GetStdout(self, cmdlist): - job = subprocess.Popen(cmdlist, stdout=subprocess.PIPE) - out = job.communicate()[0] - if job.returncode != 0: - sys.stderr.write(out + '\n') - raise GypError('Error %d running %s' % (job.returncode, cmdlist[0])) - return out.rstrip('\n') + config_settings = self.xcode_settings[configname] + xcode_archs_default = GetXcodeArchsDefault() + return xcode_archs_default.ActiveArchs( + config_settings.get('ARCHS'), + config_settings.get('VALID_ARCHS'), + config_settings.get('SDKROOT')) def _GetSdkVersionInfoItem(self, sdk, infoitem): # xcodebuild requires Xcode and can't run on Command Line Tools-only @@ -285,7 +429,7 @@ def _GetSdkVersionInfoItem(self, sdk, infoitem): # Since the CLT has no SDK paths anyway, returning None is the # most sensible route and should still do the right thing. try: - return self._GetStdout(['xcodebuild', '-version', '-sdk', sdk, infoitem]) + return GetStdout(['xcodebuild', '-version', '-sdk', sdk, infoitem]) except: pass @@ -381,6 +525,13 @@ def GetCflags(self, configname, arch=None): if self._Test('GCC_WARN_ABOUT_MISSING_NEWLINE', 'YES', default='NO'): cflags.append('-Wnewline-eof') + # In Xcode, this is only activated when GCC_COMPILER_VERSION is clang or + # llvm-gcc. It also requires a fairly recent libtool, and + # if the system clang isn't used, DYLD_LIBRARY_PATH needs to contain the + # path to the libLTO.dylib that matches the used clang. + if self._Test('LLVM_LTO', 'YES', default='NO'): + cflags.append('-flto') + self._AppendPlatformVersionMinFlags(cflags) # TODO: @@ -396,7 +547,8 @@ def GetCflags(self, configname, arch=None): if arch is not None: archs = [arch] else: - archs = self._Settings().get('ARCHS', [self._DefaultArch()]) + assert self.configname + archs = self.GetActiveArchs(self.configname) if len(archs) != 1: # TODO: Supporting fat binaries will be annoying. self._WarnUnimplemented('ARCHS') @@ -588,8 +740,8 @@ def _MapLinkerFlagFilename(self, ldflag, gyp_to_build_path): # -exported_symbols_list file # -Wl,exported_symbols_list file # -Wl,exported_symbols_list,file - LINKER_FILE = '(\S+)' - WORD = '\S+' + LINKER_FILE = r'(\S+)' + WORD = r'\S+' linker_flags = [ ['-exported_symbols_list', LINKER_FILE], # Needed for NaCl. ['-unexported_symbols_list', LINKER_FILE], @@ -653,7 +805,8 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): if arch is not None: archs = [arch] else: - archs = self._Settings().get('ARCHS', [self._DefaultArch()]) + assert self.configname + archs = self.GetActiveArchs(self.configname) if len(archs) != 1: # TODO: Supporting fat binaries will be annoying. self._WarnUnimplemented('ARCHS') @@ -678,6 +831,22 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None): for directory in framework_dirs: ldflags.append('-F' + directory.replace('$(SDKROOT)', sdk_root)) + is_extension = self._IsIosAppExtension() or self._IsIosWatchKitExtension() + if sdk_root and is_extension: + # Adds the link flags for extensions. These flags are common for all + # extensions and provide loader and main function. + # These flags reflect the compilation options used by xcode to compile + # extensions. + ldflags.append('-lpkstart') + if XcodeVersion() < '0900': + ldflags.append(sdk_root + + '/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit') + ldflags.append('-fapplication-extension') + ldflags.append('-Xlinker -rpath ' + '-Xlinker @executable_path/../../Frameworks') + + self._Appendf(ldflags, 'CLANG_CXX_LIBRARY', '-stdlib=%s') + self.configname = None return ldflags @@ -803,7 +972,7 @@ def _GetIOSPostbuilds(self, configname, output_binary): """Return a shell command to codesign the iOS output binary so it can be deployed to a device. This should be run as the very last step of the build.""" - if not (self.isIOS and self.spec['type'] == "executable"): + if not (self.isIOS and self.spec['type'] == 'executable'): return [] settings = self.xcode_settings[configname] @@ -874,65 +1043,7 @@ def AdjustLibraries(self, libraries, config_name=None): return libraries def _BuildMachineOSBuild(self): - return self._GetStdout(['sw_vers', '-buildVersion']) - - # This method ported from the logic in Homebrew's CLT version check - def _CLTVersion(self): - # pkgutil output looks like - # package-id: com.apple.pkg.CLTools_Executables - # version: 5.0.1.0.1.1382131676 - # volume: / - # location: / - # install-time: 1382544035 - # groups: com.apple.FindSystemFiles.pkg-group com.apple.DevToolsBoth.pkg-group com.apple.DevToolsNonRelocatableShared.pkg-group - STANDALONE_PKG_ID = "com.apple.pkg.DeveloperToolsCLILeo" - FROM_XCODE_PKG_ID = "com.apple.pkg.DeveloperToolsCLI" - MAVERICKS_PKG_ID = "com.apple.pkg.CLTools_Executables" - - regex = re.compile('version: (?P.+)') - for key in [MAVERICKS_PKG_ID, STANDALONE_PKG_ID, FROM_XCODE_PKG_ID]: - try: - output = self._GetStdout(['/usr/sbin/pkgutil', '--pkg-info', key]) - return re.search(regex, output).groupdict()['version'] - except: - continue - - def _XcodeVersion(self): - # `xcodebuild -version` output looks like - # Xcode 4.6.3 - # Build version 4H1503 - # or like - # Xcode 3.2.6 - # Component versions: DevToolsCore-1809.0; DevToolsSupport-1806.0 - # BuildVersion: 10M2518 - # Convert that to '0463', '4H1503'. - if len(XcodeSettings._xcode_version_cache) == 0: - try: - version_list = self._GetStdout(['xcodebuild', '-version']).splitlines() - # In some circumstances xcodebuild exits 0 but doesn't return - # the right results; for example, a user on 10.7 or 10.8 with - # a bogus path set via xcode-select - # In that case this may be a CLT-only install so fall back to - # checking that version. - if len(version_list) < 2: - raise GypError, "xcodebuild returned unexpected results" - except: - version = self._CLTVersion() - if version: - version = re.match('(\d\.\d\.?\d*)', version).groups()[0] - else: - raise GypError, "No Xcode or CLT version detected!" - # The CLT has no build information, so we return an empty string. - version_list = [version, ''] - version = version_list[0] - build = version_list[-1] - # Be careful to convert "4.2" to "0420": - version = version.split()[-1].replace('.', '') - version = (version + '0' * (3 - len(version))).zfill(4) - if build: - build = build.split()[-1] - XcodeSettings._xcode_version_cache = (version, build) - return XcodeSettings._xcode_version_cache + return GetStdout(['sw_vers', '-buildVersion']) def _XcodeIOSDeviceFamily(self, configname): family = self.xcode_settings[configname].get('TARGETED_DEVICE_FAMILY', '1') @@ -944,7 +1055,7 @@ def GetExtraPlistItems(self, configname=None): cache = {} cache['BuildMachineOSBuild'] = self._BuildMachineOSBuild() - xcode, xcode_build = self._XcodeVersion() + xcode, xcode_build = XcodeVersion() cache['DTXcode'] = xcode cache['DTXcodeBuild'] = xcode_build @@ -982,14 +1093,15 @@ def _DefaultSdkRoot(self): project, then the environment variable was empty. Starting with this version, Xcode uses the name of the newest SDK installed. """ - if self._XcodeVersion() < '0500': + xcode_version, xcode_build = XcodeVersion() + if xcode_version < '0500': return '' default_sdk_path = self._XcodeSdkPath('') default_sdk_root = XcodeSettings._sdk_root_cache.get(default_sdk_path) if default_sdk_root: return default_sdk_root try: - all_sdks = self._GetStdout(['xcodebuild', '-showsdks']) + all_sdks = GetStdout(['xcodebuild', '-showsdks']) except: # If xcodebuild fails, there will be no valid SDKs return '' @@ -1002,28 +1114,6 @@ def _DefaultSdkRoot(self): return sdk_root return '' - def _DefaultArch(self): - # For Mac projects, Xcode changed the default value used when ARCHS is not - # set from "i386" to "x86_64". - # - # For iOS projects, if ARCHS is unset, it defaults to "armv7 armv7s" when - # building for a device, and the simulator binaries are always build for - # "i386". - # - # For new projects, ARCHS is set to $(ARCHS_STANDARD_INCLUDING_64_BIT), - # which correspond to "armv7 armv7s arm64", and when building the simulator - # the architecture is either "i386" or "x86_64" depending on the simulated - # device (respectively 32-bit or 64-bit device). - # - # Since the value returned by this function is only used when ARCHS is not - # set, then on iOS we return "i386", as the default xcode project generator - # does not set ARCHS if it is not set in the .gyp file. - if self.isIOS: - return 'i386' - version, build = self._XcodeVersion() - if version >= '0500': - return 'x86_64' - return 'i386' class MacPrefixHeader(object): """A class that helps with emulating Xcode's GCC_PREFIX_HEADER feature. @@ -1131,6 +1221,81 @@ def GetPchBuildCommands(self, arch=None): ] +def XcodeVersion(): + """Returns a tuple of version and build version of installed Xcode.""" + # `xcodebuild -version` output looks like + # Xcode 4.6.3 + # Build version 4H1503 + # or like + # Xcode 3.2.6 + # Component versions: DevToolsCore-1809.0; DevToolsSupport-1806.0 + # BuildVersion: 10M2518 + # Convert that to '0463', '4H1503'. + global XCODE_VERSION_CACHE + if XCODE_VERSION_CACHE: + return XCODE_VERSION_CACHE + try: + version_list = GetStdout(['xcodebuild', '-version']).splitlines() + # In some circumstances xcodebuild exits 0 but doesn't return + # the right results; for example, a user on 10.7 or 10.8 with + # a bogus path set via xcode-select + # In that case this may be a CLT-only install so fall back to + # checking that version. + if len(version_list) < 2: + raise GypError("xcodebuild returned unexpected results") + except: + version = CLTVersion() + if version: + version = re.match(r'(\d\.\d\.?\d*)', version).groups()[0] + else: + raise GypError("No Xcode or CLT version detected!") + # The CLT has no build information, so we return an empty string. + version_list = [version, ''] + version = version_list[0] + build = version_list[-1] + # Be careful to convert "4.2" to "0420": + version = version.split()[-1].replace('.', '') + version = (version + '0' * (3 - len(version))).zfill(4) + if build: + build = build.split()[-1] + XCODE_VERSION_CACHE = (version, build) + return XCODE_VERSION_CACHE + + +# This function ported from the logic in Homebrew's CLT version check +def CLTVersion(): + """Returns the version of command-line tools from pkgutil.""" + # pkgutil output looks like + # package-id: com.apple.pkg.CLTools_Executables + # version: 5.0.1.0.1.1382131676 + # volume: / + # location: / + # install-time: 1382544035 + # groups: com.apple.FindSystemFiles.pkg-group com.apple.DevToolsBoth.pkg-group com.apple.DevToolsNonRelocatableShared.pkg-group + STANDALONE_PKG_ID = "com.apple.pkg.DeveloperToolsCLILeo" + FROM_XCODE_PKG_ID = "com.apple.pkg.DeveloperToolsCLI" + MAVERICKS_PKG_ID = "com.apple.pkg.CLTools_Executables" + + regex = re.compile('version: (?P.+)') + for key in [MAVERICKS_PKG_ID, STANDALONE_PKG_ID, FROM_XCODE_PKG_ID]: + try: + output = GetStdout(['/usr/sbin/pkgutil', '--pkg-info', key]) + return re.search(regex, output).groupdict()['version'] + except: + continue + + +def GetStdout(cmdlist): + """Returns the content of standard output returned by invoking |cmdlist|. + Raises |GypError| if the command return with a non-zero return code.""" + job = subprocess.Popen(cmdlist, stdout=subprocess.PIPE) + out = job.communicate()[0] + if job.returncode != 0: + sys.stderr.write(out + '\n') + raise GypError('Error %d running %s' % (job.returncode, cmdlist[0])) + return out.rstrip('\n') + + def MergeGlobalXcodeSettingsToSpec(global_dict, spec): """Merges the global xcode_settings dictionary into each configuration of the target represented by spec. For keys that are both in the global and the local @@ -1310,6 +1475,11 @@ def _GetXcodeEnv(xcode_settings, built_products_dir, srcroot, configuration, install_name_base = xcode_settings.GetInstallNameBase() if install_name_base: env['DYLIB_INSTALL_NAME_BASE'] = install_name_base + if XcodeVersion() >= '0500' and not env.get('SDKROOT'): + sdk_root = xcode_settings._SdkRoot(configuration) + if not sdk_root: + sdk_root = xcode_settings._XcodeSdkPath('') + env['SDKROOT'] = sdk_root if not additional_settings: additional_settings = {} @@ -1420,16 +1590,16 @@ def _HasIOSTarget(targets): def _AddIOSDeviceConfigurations(targets): """Clone all targets and append -iphoneos to the name. Configure these targets - to build for iOS devices.""" - for target_dict in targets.values(): - for config_name in target_dict['configurations'].keys(): - config = target_dict['configurations'][config_name] - new_config_name = config_name + '-iphoneos' - new_config_dict = copy.deepcopy(config) - if target_dict['toolset'] == 'target': - new_config_dict['xcode_settings']['ARCHS'] = ['armv7'] - new_config_dict['xcode_settings']['SDKROOT'] = 'iphoneos' - target_dict['configurations'][new_config_name] = new_config_dict + to build for iOS devices and use correct architectures for those builds.""" + for target_dict in targets.itervalues(): + toolset = target_dict['toolset'] + configs = target_dict['configurations'] + for config_name, config_dict in dict(configs).iteritems(): + iphoneos_config_dict = copy.deepcopy(config_dict) + configs[config_name + '-iphoneos'] = iphoneos_config_dict + configs[config_name + '-iphonesimulator'] = config_dict + if toolset == 'target': + iphoneos_config_dict['xcode_settings']['SDKROOT'] = 'iphoneos' return targets def CloneConfigurationForDeviceAndEmulator(target_dicts): diff --git a/tools/gyp/pylib/gyp/xcode_ninja.py b/tools/gyp/pylib/gyp/xcode_ninja.py new file mode 100644 index 000000000000..3820d6bf0481 --- /dev/null +++ b/tools/gyp/pylib/gyp/xcode_ninja.py @@ -0,0 +1,270 @@ +# Copyright (c) 2014 Google Inc. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Xcode-ninja wrapper project file generator. + +This updates the data structures passed to the Xcode gyp generator to build +with ninja instead. The Xcode project itself is transformed into a list of +executable targets, each with a build step to build with ninja, and a target +with every source and resource file. This appears to sidestep some of the +major performance headaches experienced using complex projects and large number +of targets within Xcode. +""" + +import errno +import gyp.generator.ninja +import os +import re +import xml.sax.saxutils + + +def _WriteWorkspace(main_gyp, sources_gyp, params): + """ Create a workspace to wrap main and sources gyp paths. """ + (build_file_root, build_file_ext) = os.path.splitext(main_gyp) + workspace_path = build_file_root + '.xcworkspace' + options = params['options'] + if options.generator_output: + workspace_path = os.path.join(options.generator_output, workspace_path) + try: + os.makedirs(workspace_path) + except OSError, e: + if e.errno != errno.EEXIST: + raise + output_string = '\n' + \ + '\n' + for gyp_name in [main_gyp, sources_gyp]: + name = os.path.splitext(os.path.basename(gyp_name))[0] + '.xcodeproj' + name = xml.sax.saxutils.quoteattr("group:" + name) + output_string += ' \n' % name + output_string += '\n' + + workspace_file = os.path.join(workspace_path, "contents.xcworkspacedata") + + try: + with open(workspace_file, 'r') as input_file: + input_string = input_file.read() + if input_string == output_string: + return + except IOError: + # Ignore errors if the file doesn't exist. + pass + + with open(workspace_file, 'w') as output_file: + output_file.write(output_string) + +def _TargetFromSpec(old_spec, params): + """ Create fake target for xcode-ninja wrapper. """ + # Determine ninja top level build dir (e.g. /path/to/out). + ninja_toplevel = None + jobs = 0 + if params: + options = params['options'] + ninja_toplevel = \ + os.path.join(options.toplevel_dir, + gyp.generator.ninja.ComputeOutputDir(params)) + jobs = params.get('generator_flags', {}).get('xcode_ninja_jobs', 0) + + target_name = old_spec.get('target_name') + product_name = old_spec.get('product_name', target_name) + product_extension = old_spec.get('product_extension') + + ninja_target = {} + ninja_target['target_name'] = target_name + ninja_target['product_name'] = product_name + if product_extension: + ninja_target['product_extension'] = product_extension + ninja_target['toolset'] = old_spec.get('toolset') + ninja_target['default_configuration'] = old_spec.get('default_configuration') + ninja_target['configurations'] = {} + + # Tell Xcode to look in |ninja_toplevel| for build products. + new_xcode_settings = {} + if ninja_toplevel: + new_xcode_settings['CONFIGURATION_BUILD_DIR'] = \ + "%s/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)" % ninja_toplevel + + if 'configurations' in old_spec: + for config in old_spec['configurations'].iterkeys(): + old_xcode_settings = \ + old_spec['configurations'][config].get('xcode_settings', {}) + if 'IPHONEOS_DEPLOYMENT_TARGET' in old_xcode_settings: + new_xcode_settings['CODE_SIGNING_REQUIRED'] = "NO" + new_xcode_settings['IPHONEOS_DEPLOYMENT_TARGET'] = \ + old_xcode_settings['IPHONEOS_DEPLOYMENT_TARGET'] + ninja_target['configurations'][config] = {} + ninja_target['configurations'][config]['xcode_settings'] = \ + new_xcode_settings + + ninja_target['mac_bundle'] = old_spec.get('mac_bundle', 0) + ninja_target['ios_app_extension'] = old_spec.get('ios_app_extension', 0) + ninja_target['ios_watchkit_extension'] = \ + old_spec.get('ios_watchkit_extension', 0) + ninja_target['ios_watchkit_app'] = old_spec.get('ios_watchkit_app', 0) + ninja_target['type'] = old_spec['type'] + if ninja_toplevel: + ninja_target['actions'] = [ + { + 'action_name': 'Compile and copy %s via ninja' % target_name, + 'inputs': [], + 'outputs': [], + 'action': [ + 'env', + 'PATH=%s' % os.environ['PATH'], + 'ninja', + '-C', + new_xcode_settings['CONFIGURATION_BUILD_DIR'], + target_name, + ], + 'message': 'Compile and copy %s via ninja' % target_name, + }, + ] + if jobs > 0: + ninja_target['actions'][0]['action'].extend(('-j', jobs)) + return ninja_target + +def IsValidTargetForWrapper(target_extras, executable_target_pattern, spec): + """Limit targets for Xcode wrapper. + + Xcode sometimes performs poorly with too many targets, so only include + proper executable targets, with filters to customize. + Arguments: + target_extras: Regular expression to always add, matching any target. + executable_target_pattern: Regular expression limiting executable targets. + spec: Specifications for target. + """ + target_name = spec.get('target_name') + # Always include targets matching target_extras. + if target_extras is not None and re.search(target_extras, target_name): + return True + + # Otherwise just show executable targets. + if spec.get('type', '') == 'executable' and \ + spec.get('product_extension', '') != 'bundle': + + # If there is a filter and the target does not match, exclude the target. + if executable_target_pattern is not None: + if not re.search(executable_target_pattern, target_name): + return False + return True + return False + +def CreateWrapper(target_list, target_dicts, data, params): + """Initialize targets for the ninja wrapper. + + This sets up the necessary variables in the targets to generate Xcode projects + that use ninja as an external builder. + Arguments: + target_list: List of target pairs: 'base/base.gyp:base'. + target_dicts: Dict of target properties keyed on target pair. + data: Dict of flattened build files keyed on gyp path. + params: Dict of global options for gyp. + """ + orig_gyp = params['build_files'][0] + for gyp_name, gyp_dict in data.iteritems(): + if gyp_name == orig_gyp: + depth = gyp_dict['_DEPTH'] + + # Check for custom main gyp name, otherwise use the default CHROMIUM_GYP_FILE + # and prepend .ninja before the .gyp extension. + generator_flags = params.get('generator_flags', {}) + main_gyp = generator_flags.get('xcode_ninja_main_gyp', None) + if main_gyp is None: + (build_file_root, build_file_ext) = os.path.splitext(orig_gyp) + main_gyp = build_file_root + ".ninja" + build_file_ext + + # Create new |target_list|, |target_dicts| and |data| data structures. + new_target_list = [] + new_target_dicts = {} + new_data = {} + + # Set base keys needed for |data|. + new_data[main_gyp] = {} + new_data[main_gyp]['included_files'] = [] + new_data[main_gyp]['targets'] = [] + new_data[main_gyp]['xcode_settings'] = \ + data[orig_gyp].get('xcode_settings', {}) + + # Normally the xcode-ninja generator includes only valid executable targets. + # If |xcode_ninja_executable_target_pattern| is set, that list is reduced to + # executable targets that match the pattern. (Default all) + executable_target_pattern = \ + generator_flags.get('xcode_ninja_executable_target_pattern', None) + + # For including other non-executable targets, add the matching target name + # to the |xcode_ninja_target_pattern| regular expression. (Default none) + target_extras = generator_flags.get('xcode_ninja_target_pattern', None) + + for old_qualified_target in target_list: + spec = target_dicts[old_qualified_target] + if IsValidTargetForWrapper(target_extras, executable_target_pattern, spec): + # Add to new_target_list. + target_name = spec.get('target_name') + new_target_name = '%s:%s#target' % (main_gyp, target_name) + new_target_list.append(new_target_name) + + # Add to new_target_dicts. + new_target_dicts[new_target_name] = _TargetFromSpec(spec, params) + + # Add to new_data. + for old_target in data[old_qualified_target.split(':')[0]]['targets']: + if old_target['target_name'] == target_name: + new_data_target = {} + new_data_target['target_name'] = old_target['target_name'] + new_data_target['toolset'] = old_target['toolset'] + new_data[main_gyp]['targets'].append(new_data_target) + + # Create sources target. + sources_target_name = 'sources_for_indexing' + sources_target = _TargetFromSpec( + { 'target_name' : sources_target_name, + 'toolset': 'target', + 'default_configuration': 'Default', + 'mac_bundle': '0', + 'type': 'executable' + }, None) + + # Tell Xcode to look everywhere for headers. + sources_target['configurations'] = {'Default': { 'include_dirs': [ depth ] } } + + sources = [] + for target, target_dict in target_dicts.iteritems(): + base = os.path.dirname(target) + files = target_dict.get('sources', []) + \ + target_dict.get('mac_bundle_resources', []) + for action in target_dict.get('actions', []): + files.extend(action.get('inputs', [])) + # Remove files starting with $. These are mostly intermediate files for the + # build system. + files = [ file for file in files if not file.startswith('$')] + + # Make sources relative to root build file. + relative_path = os.path.dirname(main_gyp) + sources += [ os.path.relpath(os.path.join(base, file), relative_path) + for file in files ] + + sources_target['sources'] = sorted(set(sources)) + + # Put sources_to_index in it's own gyp. + sources_gyp = \ + os.path.join(os.path.dirname(main_gyp), sources_target_name + ".gyp") + fully_qualified_target_name = \ + '%s:%s#target' % (sources_gyp, sources_target_name) + + # Add to new_target_list, new_target_dicts and new_data. + new_target_list.append(fully_qualified_target_name) + new_target_dicts[fully_qualified_target_name] = sources_target + new_data_target = {} + new_data_target['target_name'] = sources_target['target_name'] + new_data_target['_DEPTH'] = depth + new_data_target['toolset'] = "target" + new_data[sources_gyp] = {} + new_data[sources_gyp]['targets'] = [] + new_data[sources_gyp]['included_files'] = [] + new_data[sources_gyp]['xcode_settings'] = \ + data[orig_gyp].get('xcode_settings', {}) + new_data[sources_gyp]['targets'].append(new_data_target) + + # Write workspace to file. + _WriteWorkspace(main_gyp, sources_gyp, params) + return (new_target_list, new_target_dicts, new_data) diff --git a/tools/gyp/pylib/gyp/xcodeproj_file.py b/tools/gyp/pylib/gyp/xcodeproj_file.py index 6e4a1dc69dce..034a0d2d4fcc 100644 --- a/tools/gyp/pylib/gyp/xcodeproj_file.py +++ b/tools/gyp/pylib/gyp/xcodeproj_file.py @@ -173,7 +173,7 @@ # Used by SourceTreeAndPathFromPath -_path_leading_variable = re.compile('^\$\((.*?)\)(/(.*))?$') +_path_leading_variable = re.compile(r'^\$\((.*?)\)(/(.*))?$') def SourceTreeAndPathFromPath(input_path): """Given input_path, returns a tuple with sourceTree and path values. @@ -196,7 +196,7 @@ def SourceTreeAndPathFromPath(input_path): return (source_tree, output_path) def ConvertVariablesToShellSyntax(input_string): - return re.sub('\$\((.*?)\)', '${\\1}', input_string) + return re.sub(r'\$\((.*?)\)', '${\\1}', input_string) class XCObject(object): """The abstract base of all class types used in Xcode project files. @@ -341,13 +341,13 @@ def Copy(self): elif isinstance(value, dict): # dicts are never strong. if is_strong: - raise TypeError, 'Strong dict for key ' + key + ' in ' + \ - self.__class__.__name__ + raise TypeError('Strong dict for key ' + key + ' in ' + \ + self.__class__.__name__) else: that._properties[key] = value.copy() else: - raise TypeError, 'Unexpected type ' + value.__class__.__name__ + \ - ' for key ' + key + ' in ' + self.__class__.__name__ + raise TypeError('Unexpected type ' + value.__class__.__name__ + \ + ' for key ' + key + ' in ' + self.__class__.__name__) return that @@ -366,8 +366,7 @@ def Name(self): ('name' in self._schema and self._schema['name'][3]): return self._properties['name'] - raise NotImplementedError, \ - self.__class__.__name__ + ' must implement Name' + raise NotImplementedError(self.__class__.__name__ + ' must implement Name') def Comment(self): """Return a comment string for the object. @@ -466,10 +465,10 @@ def EnsureNoIDCollisions(self): for descendant in descendants: if descendant.id in ids: other = ids[descendant.id] - raise KeyError, \ + raise KeyError( 'Duplicate ID %s, objects "%s" and "%s" in "%s"' % \ (descendant.id, str(descendant._properties), - str(other._properties), self._properties['rootObject'].Name()) + str(other._properties), self._properties['rootObject'].Name())) ids[descendant.id] = descendant def Children(self): @@ -630,7 +629,7 @@ def _XCPrintableValue(self, tabs, value, flatten_list=False): sep printable += end_tabs + '}' else: - raise TypeError, "Can't make " + value.__class__.__name__ + ' printable' + raise TypeError("Can't make " + value.__class__.__name__ + ' printable') if comment != None: printable += ' ' + self._EncodeComment(comment) @@ -756,31 +755,31 @@ def UpdateProperties(self, properties, do_copy=False): for property, value in properties.iteritems(): # Make sure the property is in the schema. if not property in self._schema: - raise KeyError, property + ' not in ' + self.__class__.__name__ + raise KeyError(property + ' not in ' + self.__class__.__name__) # Make sure the property conforms to the schema. (is_list, property_type, is_strong) = self._schema[property][0:3] if is_list: if value.__class__ != list: - raise TypeError, \ + raise TypeError( property + ' of ' + self.__class__.__name__ + \ - ' must be list, not ' + value.__class__.__name__ + ' must be list, not ' + value.__class__.__name__) for item in value: if not isinstance(item, property_type) and \ not (item.__class__ == unicode and property_type == str): # Accept unicode where str is specified. str is treated as # UTF-8-encoded. - raise TypeError, \ + raise TypeError( 'item of ' + property + ' of ' + self.__class__.__name__ + \ ' must be ' + property_type.__name__ + ', not ' + \ - item.__class__.__name__ + item.__class__.__name__) elif not isinstance(value, property_type) and \ not (value.__class__ == unicode and property_type == str): # Accept unicode where str is specified. str is treated as # UTF-8-encoded. - raise TypeError, \ + raise TypeError( property + ' of ' + self.__class__.__name__ + ' must be ' + \ - property_type.__name__ + ', not ' + value.__class__.__name__ + property_type.__name__ + ', not ' + value.__class__.__name__) # Checks passed, perform the assignment. if do_copy: @@ -804,9 +803,9 @@ def UpdateProperties(self, properties, do_copy=False): elif isinstance(value, dict): self._properties[property] = value.copy() else: - raise TypeError, "Don't know how to copy a " + \ - value.__class__.__name__ + ' object for ' + \ - property + ' in ' + self.__class__.__name__ + raise TypeError("Don't know how to copy a " + \ + value.__class__.__name__ + ' object for ' + \ + property + ' in ' + self.__class__.__name__) else: self._properties[property] = value @@ -837,15 +836,15 @@ def AppendProperty(self, key, value): # Schema validation. if not key in self._schema: - raise KeyError, key + ' not in ' + self.__class__.__name__ + raise KeyError(key + ' not in ' + self.__class__.__name__) (is_list, property_type, is_strong) = self._schema[key][0:3] if not is_list: - raise TypeError, key + ' of ' + self.__class__.__name__ + ' must be list' + raise TypeError(key + ' of ' + self.__class__.__name__ + ' must be list') if not isinstance(value, property_type): - raise TypeError, 'item of ' + key + ' of ' + self.__class__.__name__ + \ - ' must be ' + property_type.__name__ + ', not ' + \ - value.__class__.__name__ + raise TypeError('item of ' + key + ' of ' + self.__class__.__name__ + \ + ' must be ' + property_type.__name__ + ', not ' + \ + value.__class__.__name__) # If the property doesn't exist yet, create a new empty list to receive the # item. @@ -869,7 +868,7 @@ def VerifyHasRequiredProperties(self): for property, attributes in self._schema.iteritems(): (is_list, property_type, is_strong, is_required) = attributes[0:4] if is_required and not property in self._properties: - raise KeyError, self.__class__.__name__ + ' requires ' + property + raise KeyError(self.__class__.__name__ + ' requires ' + property) def _SetDefaultsFromSchema(self): """Assign object default values according to the schema. This will not @@ -1143,16 +1142,16 @@ def _AddChildToDicts(self, child): child_path = child.PathFromSourceTreeAndPath() if child_path: if child_path in self._children_by_path: - raise ValueError, 'Found multiple children with path ' + child_path + raise ValueError('Found multiple children with path ' + child_path) self._children_by_path[child_path] = child if isinstance(child, PBXVariantGroup): child_name = child._properties.get('name', None) key = (child_name, child_path) if key in self._variant_children_by_name_and_path: - raise ValueError, 'Found multiple PBXVariantGroup children with ' + \ - 'name ' + str(child_name) + ' and path ' + \ - str(child_path) + raise ValueError('Found multiple PBXVariantGroup children with ' + \ + 'name ' + str(child_name) + ' and path ' + \ + str(child_path)) self._variant_children_by_name_and_path[key] = child def AppendChild(self, child): @@ -1508,9 +1507,12 @@ def __init__(self, properties=None, id=None, parent=None): 's': 'sourcecode.asm', 'storyboard': 'file.storyboard', 'strings': 'text.plist.strings', + 'swift': 'sourcecode.swift', 'ttf': 'file', + 'xcassets': 'folder.assetcatalog', 'xcconfig': 'text.xcconfig', 'xcdatamodel': 'wrapper.xcdatamodel', + 'xcdatamodeld':'wrapper.xcdatamodeld', 'xib': 'file.xib', 'y': 'sourcecode.yacc', } @@ -1605,7 +1607,7 @@ def ConfigurationNamed(self, name): if configuration._properties['name'] == name: return configuration - raise KeyError, name + raise KeyError(name) def DefaultConfiguration(self): """Convenience accessor to obtain the default XCBuildConfiguration.""" @@ -1662,7 +1664,7 @@ def GetBuildSetting(self, key): value = configuration_value else: if value != configuration_value: - raise ValueError, 'Variant values for ' + key + raise ValueError('Variant values for ' + key) return value @@ -1769,8 +1771,8 @@ def FileGroup(self, path): # added, either as a child or deeper descendant. The second item should # be a boolean indicating whether files should be added into hierarchical # groups or one single flat group. - raise NotImplementedError, \ - self.__class__.__name__ + ' must implement FileGroup' + raise NotImplementedError( + self.__class__.__name__ + ' must implement FileGroup') def _AddPathToDict(self, pbxbuildfile, path): """Adds path to the dict tracking paths belonging to this build phase. @@ -1779,7 +1781,7 @@ def _AddPathToDict(self, pbxbuildfile, path): """ if path in self._files_by_path: - raise ValueError, 'Found multiple build files with path ' + path + raise ValueError('Found multiple build files with path ' + path) self._files_by_path[path] = pbxbuildfile def _AddBuildFileToDicts(self, pbxbuildfile, path=None): @@ -1834,8 +1836,8 @@ def _AddBuildFileToDicts(self, pbxbuildfile, path=None): # problem. if xcfilelikeelement in self._files_by_xcfilelikeelement and \ self._files_by_xcfilelikeelement[xcfilelikeelement] != pbxbuildfile: - raise ValueError, 'Found multiple build files for ' + \ - xcfilelikeelement.Name() + raise ValueError('Found multiple build files for ' + \ + xcfilelikeelement.Name()) self._files_by_xcfilelikeelement[xcfilelikeelement] = pbxbuildfile def AppendBuildFile(self, pbxbuildfile, path=None): @@ -1999,8 +2001,8 @@ def SetDestination(self, path): subfolder = 0 relative_path = path[1:] else: - raise ValueError, 'Can\'t use path %s in a %s' % \ - (path, self.__class__.__name__) + raise ValueError('Can\'t use path %s in a %s' % \ + (path, self.__class__.__name__)) self._properties['dstPath'] = relative_path self._properties['dstSubfolderSpec'] = subfolder @@ -2236,10 +2238,16 @@ class PBXNativeTarget(XCTarget): # Mapping from Xcode product-types to settings. The settings are: # filetype : used for explicitFileType in the project file # prefix : the prefix for the file name - # suffix : the suffix for the filen ame + # suffix : the suffix for the file name _product_filetypes = { - 'com.apple.product-type.application': ['wrapper.application', - '', '.app'], + 'com.apple.product-type.application': ['wrapper.application', + '', '.app'], + 'com.apple.product-type.application.watchapp': ['wrapper.application', + '', '.app'], + 'com.apple.product-type.watchkit-extension': ['wrapper.app-extension', + '', '.appex'], + 'com.apple.product-type.app-extension': ['wrapper.app-extension', + '', '.appex'], 'com.apple.product-type.bundle': ['wrapper.cfbundle', '', '.bundle'], 'com.apple.product-type.framework': ['wrapper.framework', @@ -2312,11 +2320,11 @@ def __init__(self, properties=None, id=None, parent=None, if force_extension is not None: # If it's a wrapper (bundle), set WRAPPER_EXTENSION. + # Extension override. + suffix = '.' + force_extension if filetype.startswith('wrapper.'): self.SetBuildSetting('WRAPPER_EXTENSION', force_extension) else: - # Extension override. - suffix = '.' + force_extension self.SetBuildSetting('EXECUTABLE_EXTENSION', force_extension) if filetype.startswith('compiled.mach-o.executable'): @@ -2732,8 +2740,53 @@ def AddOrGetProjectReference(self, other_pbxproject): self._SetUpProductReferences(other_pbxproject, product_group, project_ref) + inherit_unique_symroot = self._AllSymrootsUnique(other_pbxproject, False) + targets = other_pbxproject.GetProperty('targets') + if all(self._AllSymrootsUnique(t, inherit_unique_symroot) for t in targets): + dir_path = project_ref._properties['path'] + product_group._hashables.extend(dir_path) + return [product_group, project_ref] + def _AllSymrootsUnique(self, target, inherit_unique_symroot): + # Returns True if all configurations have a unique 'SYMROOT' attribute. + # The value of inherit_unique_symroot decides, if a configuration is assumed + # to inherit a unique 'SYMROOT' attribute from its parent, if it doesn't + # define an explicit value for 'SYMROOT'. + symroots = self._DefinedSymroots(target) + for s in self._DefinedSymroots(target): + if (s is not None and not self._IsUniqueSymrootForTarget(s) or + s is None and not inherit_unique_symroot): + return False + return True if symroots else inherit_unique_symroot + + def _DefinedSymroots(self, target): + # Returns all values for the 'SYMROOT' attribute defined in all + # configurations for this target. If any configuration doesn't define the + # 'SYMROOT' attribute, None is added to the returned set. If all + # configurations don't define the 'SYMROOT' attribute, an empty set is + # returned. + config_list = target.GetProperty('buildConfigurationList') + symroots = set() + for config in config_list.GetProperty('buildConfigurations'): + setting = config.GetProperty('buildSettings') + if 'SYMROOT' in setting: + symroots.add(setting['SYMROOT']) + else: + symroots.add(None) + if len(symroots) == 1 and None in symroots: + return set() + return symroots + + def _IsUniqueSymrootForTarget(self, symroot): + # This method returns True if all configurations in target contain a + # 'SYMROOT' attribute that is unique for the given target. A value is + # unique, if the Xcode macro '$SRCROOT' appears in it in any form. + uniquifier = ['$SRCROOT', '$(SRCROOT)'] + if any(x in symroot for x in uniquifier): + return True + return False + def _SetUpProductReferences(self, other_pbxproject, product_group, project_ref): # TODO(mark): This only adds references to products in other_pbxproject @@ -2802,7 +2855,7 @@ def CompareProducts(x, y, remote_products): product_group = ref_dict['ProductGroup'] product_group._properties['children'] = sorted( product_group._properties['children'], - cmp=lambda x, y: CompareProducts(x, y, remote_products)) + cmp=lambda x, y, rp=remote_products: CompareProducts(x, y, rp)) class XCProjectFile(XCObject): @@ -2810,27 +2863,10 @@ class XCProjectFile(XCObject): _schema.update({ 'archiveVersion': [0, int, 0, 1, 1], 'classes': [0, dict, 0, 1, {}], - 'objectVersion': [0, int, 0, 1, 45], + 'objectVersion': [0, int, 0, 1, 46], 'rootObject': [0, PBXProject, 1, 1], }) - def SetXcodeVersion(self, version): - version_to_object_version = { - '2.4': 45, - '3.0': 45, - '3.1': 45, - '3.2': 46, - } - if not version in version_to_object_version: - supported_str = ', '.join(sorted(version_to_object_version.keys())) - raise Exception( - 'Unsupported Xcode version %s (supported: %s)' % - ( version, supported_str ) ) - compatibility_version = 'Xcode %s' % version - self._properties['rootObject'].SetProperty('compatibilityVersion', - compatibility_version) - self.SetProperty('objectVersion', version_to_object_version[version]); - def ComputeIDs(self, recursive=True, overwrite=True, hash=None): # Although XCProjectFile is implemented here as an XCObject, it's not a # proper object in the Xcode sense, and it certainly doesn't have its own diff --git a/tools/gyp/pylintrc b/tools/gyp/pylintrc deleted file mode 100644 index d7c23d2a211d..000000000000 --- a/tools/gyp/pylintrc +++ /dev/null @@ -1,307 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Profiled execution. -profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -# C0103: Invalid name "NN" (should match [a-z_][a-z0-9_]{2,30}$) -# C0111: Missing docstring -# C0302: Too many lines in module (NN) -# R0902: Too many instance attributes (N/7) -# R0903: Too few public methods (N/2) -# R0904: Too many public methods (NN/20) -# R0912: Too many branches (NN/12) -# R0913: Too many arguments (N/5) -# R0914: Too many local variables (NN/15) -# R0915: Too many statements (NN/50) -# W0141: Used builtin function 'map' -# W0142: Used * or ** magic -# W0232: Class has no __init__ method -# W0511: TODO -# W0603: Using the global statement -# -# These should be enabled eventually: -# C0112: Empty docstring -# C0301: Line too long (NN/80) -# C0321: More than one statement on single line -# C0322: Operator not preceded by a space -# C0323: Operator not followed by a space -# C0324: Comma not followed by a space -# E0101: Explicit return in __init__ -# E0102: function already defined line NN -# E1002: Use of super on an old style class -# E1101: Instance of 'XX' has no 'YY' member -# E1103: Instance of 'XX' has no 'XX' member (but some types could not be inferred) -# E0602: Undefined variable 'XX' -# F0401: Unable to import 'XX' -# R0201: Method could be a function -# R0801: Similar lines in N files -# W0102: Dangerous default value {} as argument -# W0104: Statement seems to have no effect -# W0105: String statement has no effect -# W0108: Lambda may not be necessary -# W0201: Attribute 'XX' defined outside __init__ -# W0212: Access to a protected member XX of a client class -# W0221: Arguments number differs from overridden method -# W0223: Method 'XX' is abstract in class 'YY' but is not overridden -# W0231: __init__ method from base class 'XX' is not called -# W0301: Unnecessary semicolon -# W0311: Bad indentation. Found NN spaces, expected NN -# W0401: Wildcard import XX -# W0402: Uses of a deprecated module 'string' -# W0403: Relative import 'XX', should be 'YY.XX' -# W0404: Reimport 'XX' (imported line NN) -# W0601: Global variable 'XX' undefined at the module level -# W0602: Using global for 'XX' but no assignment is done -# W0611: Unused import pprint -# W0612: Unused variable 'XX' -# W0613: Unused argument 'XX' -# W0614: Unused import XX from wildcard import -# W0621: Redefining name 'XX' from outer scope (line NN) -# W0622: Redefining built-in 'NN' -# W0631: Using possibly undefined loop variable 'XX' -# W0701: Raising a string exception -# W0702: No exception type(s) specified -disable=C0103,C0111,C0302,R0902,R0903,R0904,R0912,R0913,R0914,R0915,W0141,W0142,W0232,W0511,W0603,C0112,C0301,C0321,C0322,C0323,C0324,E0101,E0102,E1002,E1101,E1103,E0602,F0401,R0201,R0801,W0102,W0104,W0105,W0108,W0201,W0212,W0221,W0223,W0231,W0301,W0311,W0401,W0402,W0403,W0404,W0601,W0602,W0611,W0612,W0613,W0614,W0621,W0622,W0631,W0701,W0702 - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=text - -# Include message's id in output -include-ids=yes - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -comment=no - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - - -[BASIC] - -# Required attributes for module, separated by a comma -required-attributes= - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/tools/gyp/tools/emacs/gyp.el b/tools/gyp/tools/emacs/gyp.el index 3db9f6459434..b98b155cedad 100644 --- a/tools/gyp/tools/emacs/gyp.el +++ b/tools/gyp/tools/emacs/gyp.el @@ -15,14 +15,36 @@ "recent emacsen), not from the older and less maintained " "python-mode.el"))) -(defadvice python-calculate-indentation (after ami-outdent-closing-parens - activate) +(defadvice python-indent-calculate-levels (after gyp-outdent-closing-parens + activate) "De-indent closing parens, braces, and brackets in gyp-mode." - (if (and (eq major-mode 'gyp-mode) - (string-match "^ *[])}][],)}]* *$" - (buffer-substring-no-properties - (line-beginning-position) (line-end-position)))) - (setq ad-return-value (- ad-return-value 2)))) + (when (and (eq major-mode 'gyp-mode) + (string-match "^ *[])}][],)}]* *$" + (buffer-substring-no-properties + (line-beginning-position) (line-end-position)))) + (setf (first python-indent-levels) + (- (first python-indent-levels) python-continuation-offset)))) + +(defadvice python-indent-guess-indent-offset (around + gyp-indent-guess-indent-offset + activate) + "Guess correct indent offset in gyp-mode." + (or (and (not (eq major-mode 'gyp-mode)) + ad-do-it) + (save-excursion + (save-restriction + (widen) + (goto-char (point-min)) + ;; Find first line ending with an opening brace that is not a comment. + (or (and (re-search-forward "\\(^[[{]$\\|^.*[^#].*[[{]$\\)") + (forward-line) + (/= (current-indentation) 0) + (set (make-local-variable 'python-indent-offset) + (current-indentation)) + (set (make-local-variable 'python-continuation-offset) + (current-indentation))) + (message "Can't guess gyp indent offset, using default: %s" + python-continuation-offset)))))) (define-derived-mode gyp-mode python-mode "Gyp" "Major mode for editing .gyp files. See http://code.google.com/p/gyp/" @@ -35,9 +57,10 @@ (defun gyp-set-indentation () "Hook function to configure python indentation to suit gyp mode." - (setq python-continuation-offset 2 - python-indent 2 - python-guess-indent nil)) + (set (make-local-variable 'python-indent-offset) 2) + (set (make-local-variable 'python-continuation-offset) 2) + (set (make-local-variable 'python-indent-guess-indent-offset) t) + (python-indent-guess-indent-offset)) (add-hook 'gyp-mode-hook 'gyp-set-indentation) @@ -218,11 +241,11 @@ ;; Top-level keywords (list (concat "['\"]\\(" (regexp-opt (list "action" "action_name" "actions" "cflags" - "conditions" "configurations" "copies" "defines" - "dependencies" "destination" + "cflags_cc" "conditions" "configurations" + "copies" "defines" "dependencies" "destination" "direct_dependent_settings" "export_dependent_settings" "extension" "files" - "include_dirs" "includes" "inputs" "libraries" + "include_dirs" "includes" "inputs" "ldflags" "libraries" "link_settings" "mac_bundle" "message" "msvs_external_rule" "outputs" "product_name" "process_outputs_as_sources" "rules" "rule_name" diff --git a/tools/gyp/tools/pretty_sln.py b/tools/gyp/tools/pretty_sln.py index 3195d8581beb..ca8cf4ad3fb8 100755 --- a/tools/gyp/tools/pretty_sln.py +++ b/tools/gyp/tools/pretty_sln.py @@ -38,12 +38,13 @@ def ParseSolution(solution_file): # Regular expressions that matches the SLN format. # The first line of a project definition. - begin_project = re.compile(('^Project\("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942' - '}"\) = "(.*)", "(.*)", "(.*)"$')) + begin_project = re.compile(r'^Project\("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942' + r'}"\) = "(.*)", "(.*)", "(.*)"$') # The last line of a project definition. end_project = re.compile('^EndProject$') # The first line of a dependency list. - begin_dep = re.compile('ProjectSection\(ProjectDependencies\) = postProject$') + begin_dep = re.compile( + r'ProjectSection\(ProjectDependencies\) = postProject$') # The last line of a dependency list. end_dep = re.compile('EndProjectSection$') # A line describing a dependency.