From 66807e995de9a16b1cfe035f76958a178c381854 Mon Sep 17 00:00:00 2001 From: Richard Townsend Date: Tue, 24 Mar 2020 19:00:57 +0000 Subject: [PATCH] tools,gyp: add support for MSVC cross-compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change means that GYP can now generate two sets of projects: one exclusively for a host x64 machine and one containing a mix of x64 and Arm targets. The names of host targets are fixed up to end with _host.exe, and any actions involving them are fixed up. This allows compilation of Node on an x64 server for a Windows on Arm target. PR-URL: https://github.com/nodejs/node/pull/32867 Reviewed-By: Christian Clauss Reviewed-By: João Reis --- tools/gyp/pylib/gyp/generator/msvs.py | 119 ++++++++++++++++++-------- 1 file changed, 81 insertions(+), 38 deletions(-) diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index 933042c7113c59..09abadb1bcb658 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -40,6 +40,7 @@ # letters. VALID_MSVS_GUID_CHARS = re.compile(r'^[A-F0-9\-]+$') +generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested() generator_default_variables = { 'DRIVER_PREFIX': '', @@ -51,7 +52,7 @@ 'STATIC_LIB_SUFFIX': '.lib', 'SHARED_LIB_SUFFIX': '.dll', 'INTERMEDIATE_DIR': '$(IntDir)', - 'SHARED_INTERMEDIATE_DIR': '$(OutDir)obj/global_intermediate', + 'SHARED_INTERMEDIATE_DIR': '$(OutDir)/obj/global_intermediate', 'OS': 'win', 'PRODUCT_DIR': '$(OutDir)', 'LIB_DIR': '$(OutDir)lib', @@ -286,7 +287,7 @@ def _ConfigTargetVersion(config_data): return config_data.get('msvs_target_version', 'Windows7') -def _ConfigPlatform(config_data): +def _ConfigPlatform(config_data, spec): return config_data.get('msvs_configuration_platform', 'Win32') @@ -297,8 +298,8 @@ def _ConfigBaseName(config_name, platform_name): return config_name -def _ConfigFullName(config_name, config_data): - platform_name = _ConfigPlatform(config_data) +def _ConfigFullName(config_name, config_data, spec): + platform_name = _ConfigPlatform(config_data, spec) return '%s|%s' % (_ConfigBaseName(config_name, platform_name), platform_name) @@ -951,7 +952,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version): return toolset -def _GenerateProject(project, options, version, generator_flags): +def _GenerateProject(project, options, version, generator_flags, spec): """Generates a vcproj file. Arguments: @@ -969,7 +970,7 @@ def _GenerateProject(project, options, version, generator_flags): return [] if version.UsesVcxproj(): - return _GenerateMSBuildProject(project, options, version, generator_flags) + return _GenerateMSBuildProject(project, options, version, generator_flags, spec) else: return _GenerateMSVSProject(project, options, version, generator_flags) @@ -1091,7 +1092,7 @@ def _GetUniquePlatforms(spec): # Gather list of unique platforms. platforms = OrderedSet() for configuration in spec['configurations']: - platforms.add(_ConfigPlatform(spec['configurations'][configuration])) + platforms.add(_ConfigPlatform(spec['configurations'][configuration], spec)) platforms = list(platforms) return platforms @@ -1801,6 +1802,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat): # Convert into a tree of dicts on path. for p in sln_projects: gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2] + if p.endswith("#host"): + target += "_host" gyp_dir = os.path.dirname(gyp_file) path_dict = _GetPathDict(root, gyp_dir) path_dict[target + '.vcproj'] = project_objects[p] @@ -1819,7 +1822,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version): default_config = _GetDefaultConfiguration(spec) proj_filename = default_config.get('msvs_existing_vcproj') if not proj_filename: - proj_filename = (spec['target_name'] + options.suffix + + proj_filename = spec['target_name'] + if spec['toolset'] == 'host': + proj_filename += "_host" + proj_filename = (proj_filename + options.suffix + msvs_version.ProjectExtension()) build_file = gyp.common.BuildFile(qualified_target) @@ -1838,10 +1844,12 @@ def _GetPlatformOverridesOfProject(spec): # solution configurations for this target. config_platform_overrides = {} for config_name, c in spec['configurations'].items(): - config_fullname = _ConfigFullName(config_name, c) - platform = c.get('msvs_target_platform', _ConfigPlatform(c)) + config_fullname = _ConfigFullName(config_name, c, spec) + platform = c.get('msvs_target_platform', _ConfigPlatform(c, spec)) fixed_config_fullname = '%s|%s' % ( - _ConfigBaseName(config_name, _ConfigPlatform(c)), platform) + _ConfigBaseName(config_name, _ConfigPlatform(c, spec)), platform) + if spec['toolset'] == 'host' and generator_supports_multiple_toolsets: + fixed_config_fullname = '%s|x64' % (config_name,) config_platform_overrides[config_fullname] = fixed_config_fullname return config_platform_overrides @@ -1862,19 +1870,18 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version): projects = {} for qualified_target in target_list: spec = target_dicts[qualified_target] - if spec['toolset'] != 'target': - raise GypError( - 'Multiple toolsets not supported in msvs build (target %s)' % - qualified_target) proj_path, fixpath_prefix = _GetPathOfProject(qualified_target, spec, options, msvs_version) guid = _GetGuidOfProject(proj_path, spec) overrides = _GetPlatformOverridesOfProject(spec) build_file = gyp.common.BuildFile(qualified_target) # Create object for this project. + target_name = spec['target_name'] + if spec['toolset'] == 'host': + target_name += '_host' obj = MSVSNew.MSVSProject( proj_path, - name=spec['target_name'], + name=target_name, guid=guid, spec=spec, build_file=build_file, @@ -2041,7 +2048,10 @@ def GenerateOutput(target_list, target_dicts, data, params): for qualified_target in target_list: spec = target_dicts[qualified_target] for config_name, config in spec['configurations'].items(): - configs.add(_ConfigFullName(config_name, config)) + config_name = _ConfigFullName(config_name, config, spec) + configs.add(config_name) + if config_name == 'Release|arm64': + configs.add("Release|x64") configs = list(configs) # Figure out all the projects that will be generated and their guids @@ -2053,11 +2063,14 @@ def GenerateOutput(target_list, target_dicts, data, params): for project in project_objects.values(): fixpath_prefix = project.fixpath_prefix missing_sources.extend(_GenerateProject(project, options, msvs_version, - generator_flags)) + generator_flags, spec)) fixpath_prefix = None for build_file in data: # Validate build_file extension + target_only_configs = configs + if generator_supports_multiple_toolsets: + target_only_configs = [i for i in configs if i.endswith('arm64')] if not build_file.endswith('.gyp'): continue sln_path = os.path.splitext(build_file)[0] + options.suffix + '.sln' @@ -2072,7 +2085,7 @@ def GenerateOutput(target_list, target_dicts, data, params): # Create solution. sln = MSVSNew.MSVSSolution(sln_path, entries=root_entries, - variants=configs, + variants=target_only_configs, websiteProperties=False, version=msvs_version) sln.Write() @@ -2674,21 +2687,23 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules): easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True) -def _GetConfigurationAndPlatform(name, settings): +def _GetConfigurationAndPlatform(name, settings, spec): configuration = name.rsplit('_', 1)[0] platform = settings.get('msvs_configuration_platform', 'Win32') + if spec['toolset'] == 'host' and platform == 'arm64': + platform = 'x64' # Host-only tools are always built for x64 return (configuration, platform) -def _GetConfigurationCondition(name, settings): +def _GetConfigurationCondition(name, settings, spec): return (r"'$(Configuration)|$(Platform)'=='%s|%s'" % - _GetConfigurationAndPlatform(name, settings)) + _GetConfigurationAndPlatform(name, settings, spec)) -def _GetMSBuildProjectConfigurations(configurations): +def _GetMSBuildProjectConfigurations(configurations, spec): group = ['ItemGroup', {'Label': 'ProjectConfigurations'}] for (name, settings) in sorted(configurations.items()): - configuration, platform = _GetConfigurationAndPlatform(name, settings) + configuration, platform = _GetConfigurationAndPlatform(name, settings, spec) designation = '%s|%s' % (configuration, platform) group.append( ['ProjectConfiguration', {'Include': designation}, @@ -2740,7 +2755,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name): platform_name = None msvs_windows_sdk_version = None for configuration in spec['configurations'].values(): - platform_name = platform_name or _ConfigPlatform(configuration) + platform_name = platform_name or _ConfigPlatform(configuration, spec) msvs_windows_sdk_version = (msvs_windows_sdk_version or _ConfigWindowsTargetPlatformVersion(configuration, version)) if platform_name and msvs_windows_sdk_version: @@ -2762,7 +2777,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file): properties = {} for name, settings in spec['configurations'].items(): msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file) - condition = _GetConfigurationCondition(name, settings) + condition = _GetConfigurationCondition(name, settings, spec) character_set = msbuild_attributes.get('CharacterSet') config_type = msbuild_attributes.get('ConfigurationType') _AddConditionalProperty(properties, condition, 'ConfigurationType', @@ -2790,12 +2805,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset): return properties -def _GetMSBuildPropertySheets(configurations): +def _GetMSBuildPropertySheets(configurations, spec): user_props = r'$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props' additional_props = {} props_specified = False for name, settings in sorted(configurations.items()): - configuration = _GetConfigurationCondition(name, settings) + configuration = _GetConfigurationCondition(name, settings, spec) if 'msbuild_props' in settings: additional_props[configuration] = _FixPaths(settings['msbuild_props']) props_specified = True @@ -2946,7 +2961,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file): properties = {} for (name, configuration) in sorted(configurations.items()): - condition = _GetConfigurationCondition(name, configuration) + condition = _GetConfigurationCondition(name, configuration, spec) attributes = _GetMSBuildAttributes(spec, configuration, build_file) msbuild_settings = configuration['finalized_msbuild_settings'] _AddConditionalProperty(properties, condition, 'IntDir', @@ -3055,7 +3070,9 @@ def _GetMSBuildToolSettingsSections(spec, configurations): for (name, configuration) in sorted(configurations.items()): msbuild_settings = configuration['finalized_msbuild_settings'] group = ['ItemDefinitionGroup', - {'Condition': _GetConfigurationCondition(name, configuration)} + {'Condition': + _GetConfigurationCondition(name, configuration, spec) + } ] for tool_name, tool_settings in sorted(msbuild_settings.items()): # Skip the tool named '' which is a holder of global settings handled @@ -3277,7 +3294,9 @@ def _AddSources2(spec, sources, exclusions, grouped_sources, extensions_excluded_from_precompile = ['.c'] if precompiled_source == source: - condition = _GetConfigurationCondition(config_name, configuration) + condition = _GetConfigurationCondition( + config_name, configuration, spec + ) detail.append(['PrecompiledHeader', {'Condition': condition}, 'Create' @@ -3296,12 +3315,26 @@ def _AddSources2(spec, sources, exclusions, grouped_sources, _GetUniquePlatforms(spec)) grouped_sources[group].append([element, {'Include': source}] + detail) - -def _GetMSBuildProjectReferences(project): +def _GetMSBuildProjectReferences(project, spec): + current_configuration = spec['default_configuration'] references = [] if project.dependencies: group = ['ItemGroup'] + added_dependency_set = set() for dependency in project.dependencies: + dependency_spec = dependency.spec + should_skip_dep = False + if project.spec["toolset"] == 'target': + if dependency_spec['toolset'] == 'host': + if dependency_spec['type'] == 'static_library': + should_skip_dep = True + if dependency.name.startswith('run_'): + should_skip_dep = False + if should_skip_dep: + continue + + canonical_name = dependency.name.replace('_host', '') + added_dependency_set.add(canonical_name) guid = dependency.guid project_dir = os.path.split(project.path)[0] relative_path = gyp.common.RelativePath(dependency.path, project_dir) @@ -3323,7 +3356,7 @@ def _GetMSBuildProjectReferences(project): return references -def _GenerateMSBuildProject(project, options, version, generator_flags): +def _GenerateMSBuildProject(project, options, version, generator_flags, spec): spec = project.spec configurations = spec['configurations'] project_dir, project_file_name = os.path.split(project.path) @@ -3411,7 +3444,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): 'DefaultTargets': 'Build' }] - content += _GetMSBuildProjectConfigurations(configurations) + content += _GetMSBuildProjectConfigurations(configurations, spec) content += _GetMSBuildGlobalProperties(spec, version, project.guid, project_file_name) content += import_default_section @@ -3422,10 +3455,10 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): content += _GetMSBuildLocalProperties(project.msbuild_toolset) content += import_cpp_props_section content += import_masm_props_section - if spec.get('msvs_enable_marmasm'): + if spec.get('msvs_enable_marmasm') or True: content += import_marmasm_props_section content += _GetMSBuildExtensions(props_files_of_rules) - content += _GetMSBuildPropertySheets(configurations) + content += _GetMSBuildPropertySheets(configurations, spec) content += macro_section content += _GetMSBuildConfigurationGlobalProperties(spec, configurations, project.build_file) @@ -3433,7 +3466,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): content += _GetMSBuildSources( spec, sources, exclusions, rule_dependencies, extension_to_rule_name, actions_spec, sources_handled_by_action, list_excluded) - content += _GetMSBuildProjectReferences(project) + content += _GetMSBuildProjectReferences(project, spec) content += import_cpp_targets_section content += import_masm_targets_section if spec.get('msvs_enable_marmasm'): @@ -3516,15 +3549,25 @@ def _GenerateActionsForMSBuild(spec, actions_to_add): sources_handled_by_action = OrderedSet() actions_spec = [] for primary_input, actions in actions_to_add.items(): + if generator_supports_multiple_toolsets: + primary_input = primary_input.replace(".exe", "_host.exe") inputs = OrderedSet() outputs = OrderedSet() descriptions = [] commands = [] for action in actions: + def fixup_host_exe(i): + if "$(OutDir)" in i: + i = i.replace('.exe', '_host.exe') + return i + if generator_supports_multiple_toolsets: + action['inputs'] = [fixup_host_exe(i) for i in action['inputs']] inputs.update(OrderedSet(action['inputs'])) outputs.update(OrderedSet(action['outputs'])) descriptions.append(action['description']) cmd = action['command'] + if generator_supports_multiple_toolsets: + cmd = cmd.replace('.exe', "_host.exe") # For most actions, add 'call' so that actions that invoke batch files # return and continue executing. msbuild_use_call provides a way to # disable this but I have not seen any adverse effect from doing that