build: support apple silicon (arm64 darwin) builds

gyp/.github/workflows/Python_tests.yml

@@ -14,9 +14,9 @@ jobs:
         os: [macos-latest, ubuntu-latest] # , windows-latest]
         python-version: [2.7, 3.6, 3.7, 3.8] # 3.5,
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

gyp/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [Unreleased]
+
+## [0.4.0] - 2020-07-14
+
+### Added
+- Added support for passing arbitrary architectures to Xcode builds, enables `arm64` builds.
+
+### Fixed
+- Fixed a bug on Solaris where copying archives failed.
+
+## [0.3.0] - 2020-06-06
+
+### Added
+- Added support for MSVC cross-compilation. This allows compilation on x64 for
+  a Windows ARM target.
+
+### Fixed
+- Fixed XCode CLT version detection on macOS Catalina.
+
+## [0.2.1] - 2020-05-05
+
+### Fixed
+- Relicensed to Node.js contributors.
+- Fixed Windows bug introduced in v0.2.0.
+
+## [0.2.0] - 2020-04-06
+
+This is the first release of this project, based on https://chromium.googlesource.com/external/gyp
+with changes made over the years in Node.js and node-gyp.
+
+[Unreleased]: https://github.com/nodejs/gyp-next/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/nodejs/gyp-next/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/nodejs/gyp-next/compare/v0.2.1...v0.3.0
+[0.2.1]: https://github.com/nodejs/gyp-next/compare/v0.2.0...v0.2.1
+[0.2.0]: https://github.com/nodejs/gyp-next/releases/tag/v0.2.0

gyp/pylib/gyp/generator/android.py

@@ -981,9 +981,9 @@ def WriteList(
     """
         values = ""
         if value_list:
-            value_list = [quoter(prefix + l) for l in value_list]
+            value_list = [quoter(prefix + value) for value in value_list]
             if local_pathify:
-                value_list = [self.LocalPathify(l) for l in value_list]
+                value_list = [self.LocalPathify(value) for value in value_list]
             values = " \\\n\t" + " \\\n\t".join(value_list)
         self.fp.write("%s :=%s\n\n" % (variable, values))
 

gyp/pylib/gyp/generator/make.py

@@ -1342,7 +1342,7 @@ def WriteSources(
             )
 
             if self.flavor == "mac":
-                cflags = self.xcode_settings.GetCflags(configname)
+                cflags = self.xcode_settings.GetCflags(configname, arch=config.get('xcode_configuration_platform'))
                 cflags_c = self.xcode_settings.GetCflagsC(configname)
                 cflags_cc = self.xcode_settings.GetCflagsCC(configname)
                 cflags_objc = self.xcode_settings.GetCflagsObjC(configname)
@@ -1637,6 +1637,7 @@ def WriteTarget(
                         configname,
                         generator_default_variables["PRODUCT_DIR"],
                         lambda p: Sourceify(self.Absolutify(p)),
+                        arch=config.get('xcode_configuration_platform')
                     )
 
                     # TARGET_POSTBUILDS_$(BUILDTYPE) is added to postbuilds later on.
@@ -1944,7 +1945,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar
     """
         values = ""
         if value_list:
-            value_list = [quoter(prefix + l) for l in value_list]
+            value_list = [quoter(prefix + value) for value in value_list]
             values = " \\\n\t" + " \\\n\t".join(value_list)
         self.fp.write("%s :=%s\n\n" % (variable, values))
 
@@ -2362,7 +2363,14 @@ def CalculateMakefilePath(build_file, base_name):
             }
         )
     elif flavor == "solaris":
-        header_params.update({"flock": "./gyp-flock-tool flock", "flock_index": 2})
+        copy_archive_arguments = "-pPRf@"
+        header_params.update(
+            {
+                "copy_archive_args": copy_archive_arguments,
+                "flock": "./gyp-flock-tool flock",
+                "flock_index": 2
+            }
+        )
     elif flavor == "freebsd":
         # Note: OpenBSD has sysutils/flock. lockf seems to be FreeBSD specific.
         header_params.update({"flock": "lockf"})

gyp/pylib/gyp/generator/msvs.py

@@ -39,6 +39,7 @@
 # letters.
 VALID_MSVS_GUID_CHARS = re.compile(r"^[A-F0-9\-]+$")
 
+generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()
 
 generator_default_variables = {
     "DRIVER_PREFIX": "",
@@ -50,7 +51,7 @@
     "STATIC_LIB_SUFFIX": ".lib",
     "SHARED_LIB_SUFFIX": ".dll",
     "INTERMEDIATE_DIR": "$(IntDir)",
-    "SHARED_INTERMEDIATE_DIR": "$(OutDir)obj/global_intermediate",
+    "SHARED_INTERMEDIATE_DIR": "$(OutDir)/obj/global_intermediate",
     "OS": "win",
     "PRODUCT_DIR": "$(OutDir)",
     "LIB_DIR": "$(OutDir)lib",
@@ -1005,7 +1006,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
     return toolset
 
 
-def _GenerateProject(project, options, version, generator_flags):
+def _GenerateProject(project, options, version, generator_flags, spec):
     """Generates a vcproj file.
 
   Arguments:
@@ -1023,7 +1024,7 @@ def _GenerateProject(project, options, version, generator_flags):
         return []
 
     if version.UsesVcxproj():
-        return _GenerateMSBuildProject(project, options, version, generator_flags)
+        return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
     else:
         return _GenerateMSVSProject(project, options, version, generator_flags)
 
@@ -1903,6 +1904,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
     # Convert into a tree of dicts on path.
     for p in sln_projects:
         gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
+        if p.endswith("#host"):
+            target += "_host"
         gyp_dir = os.path.dirname(gyp_file)
         path_dict = _GetPathDict(root, gyp_dir)
         path_dict[target + ".vcproj"] = project_objects[p]
@@ -1921,9 +1924,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
     default_config = _GetDefaultConfiguration(spec)
     proj_filename = default_config.get("msvs_existing_vcproj")
     if not proj_filename:
-        proj_filename = (
-            spec["target_name"] + options.suffix + msvs_version.ProjectExtension()
-        )
+        proj_filename = spec["target_name"]
+        if spec["toolset"] == "host":
+            proj_filename += "_host"
+        proj_filename = proj_filename + options.suffix + msvs_version.ProjectExtension()
 
     build_file = gyp.common.BuildFile(qualified_target)
     proj_path = os.path.join(os.path.dirname(build_file), proj_filename)
@@ -1948,6 +1952,8 @@ def _GetPlatformOverridesOfProject(spec):
             _ConfigBaseName(config_name, _ConfigPlatform(c)),
             platform,
         )
+        if spec["toolset"] == "host" and generator_supports_multiple_toolsets:
+            fixed_config_fullname = "%s|x64" % (config_name,)
         config_platform_overrides[config_fullname] = fixed_config_fullname
     return config_platform_overrides
 
@@ -1968,21 +1974,19 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
     projects = {}
     for qualified_target in target_list:
         spec = target_dicts[qualified_target]
-        if spec["toolset"] != "target":
-            raise GypError(
-                "Multiple toolsets not supported in msvs build (target %s)"
-                % qualified_target
-            )
         proj_path, fixpath_prefix = _GetPathOfProject(
             qualified_target, spec, options, msvs_version
         )
         guid = _GetGuidOfProject(proj_path, spec)
         overrides = _GetPlatformOverridesOfProject(spec)
         build_file = gyp.common.BuildFile(qualified_target)
         # Create object for this project.
+        target_name = spec["target_name"]
+        if spec["toolset"] == "host":
+            target_name += "_host"
         obj = MSVSNew.MSVSProject(
             proj_path,
-            name=spec["target_name"],
+            name=target_name,
             guid=guid,
             spec=spec,
             build_file=build_file,
@@ -2161,7 +2165,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
     for qualified_target in target_list:
         spec = target_dicts[qualified_target]
         for config_name, config in spec["configurations"].items():
-            configs.add(_ConfigFullName(config_name, config))
+            config_name = _ConfigFullName(config_name, config)
+            configs.add(config_name)
+            if config_name == "Release|arm64":
+                configs.add("Release|x64")
     configs = list(configs)
 
     # Figure out all the projects that will be generated and their guids
@@ -2174,12 +2181,15 @@ def GenerateOutput(target_list, target_dicts, data, params):
     for project in project_objects.values():
         fixpath_prefix = project.fixpath_prefix
         missing_sources.extend(
-            _GenerateProject(project, options, msvs_version, generator_flags)
+            _GenerateProject(project, options, msvs_version, generator_flags, spec)
         )
     fixpath_prefix = None
 
     for build_file in data:
         # Validate build_file extension
+        target_only_configs = configs
+        if generator_supports_multiple_toolsets:
+            target_only_configs = [i for i in configs if i.endswith("arm64")]
         if not build_file.endswith(".gyp"):
             continue
         sln_path = os.path.splitext(build_file)[0] + options.suffix + ".sln"
@@ -2196,7 +2206,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
         sln = MSVSNew.MSVSSolution(
             sln_path,
             entries=root_entries,
-            variants=configs,
+            variants=target_only_configs,
             websiteProperties=False,
             version=msvs_version,
         )
@@ -2930,22 +2940,24 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
     easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)
 
 
-def _GetConfigurationAndPlatform(name, settings):
+def _GetConfigurationAndPlatform(name, settings, spec):
     configuration = name.rsplit("_", 1)[0]
     platform = settings.get("msvs_configuration_platform", "Win32")
+    if spec["toolset"] == "host" and platform == "arm64":
+        platform = "x64"  # Host-only tools are always built for x64
     return (configuration, platform)
 
 
-def _GetConfigurationCondition(name, settings):
+def _GetConfigurationCondition(name, settings, spec):
     return r"'$(Configuration)|$(Platform)'=='%s|%s'" % _GetConfigurationAndPlatform(
-        name, settings
+        name, settings, spec
     )
 
 
-def _GetMSBuildProjectConfigurations(configurations):
+def _GetMSBuildProjectConfigurations(configurations, spec):
     group = ["ItemGroup", {"Label": "ProjectConfigurations"}]
     for (name, settings) in sorted(configurations.items()):
-        configuration, platform = _GetConfigurationAndPlatform(name, settings)
+        configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
         designation = "%s|%s" % (configuration, platform)
         group.append(
             [
@@ -3033,7 +3045,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
     properties = {}
     for name, settings in spec["configurations"].items():
         msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
-        condition = _GetConfigurationCondition(name, settings)
+        condition = _GetConfigurationCondition(name, settings, spec)
         character_set = msbuild_attributes.get("CharacterSet")
         config_type = msbuild_attributes.get("ConfigurationType")
         _AddConditionalProperty(properties, condition, "ConfigurationType", config_type)
@@ -3064,12 +3076,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
     return properties
 
 
-def _GetMSBuildPropertySheets(configurations):
+def _GetMSBuildPropertySheets(configurations, spec):
     user_props = r"$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
     additional_props = {}
     props_specified = False
     for name, settings in sorted(configurations.items()):
-        configuration = _GetConfigurationCondition(name, settings)
+        configuration = _GetConfigurationCondition(name, settings, spec)
         if "msbuild_props" in settings:
             additional_props[configuration] = _FixPaths(settings["msbuild_props"])
             props_specified = True
@@ -3222,7 +3234,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
 
     properties = {}
     for (name, configuration) in sorted(configurations.items()):
-        condition = _GetConfigurationCondition(name, configuration)
+        condition = _GetConfigurationCondition(name, configuration, spec)
         attributes = _GetMSBuildAttributes(spec, configuration, build_file)
         msbuild_settings = configuration["finalized_msbuild_settings"]
         _AddConditionalProperty(
@@ -3345,7 +3357,7 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
         msbuild_settings = configuration["finalized_msbuild_settings"]
         group = [
             "ItemDefinitionGroup",
-            {"Condition": _GetConfigurationCondition(name, configuration)},
+            {"Condition": _GetConfigurationCondition(name, configuration, spec)},
         ]
         for tool_name, tool_settings in sorted(msbuild_settings.items()):
             # Skip the tool named '' which is a holder of global settings handled
@@ -3625,7 +3637,7 @@ def _AddSources2(
 
                     if precompiled_source == source:
                         condition = _GetConfigurationCondition(
-                            config_name, configuration
+                            config_name, configuration, spec
                         )
                         detail.append(
                             ["PrecompiledHeader", {"Condition": condition}, "Create"]
@@ -3652,7 +3664,21 @@ def _GetMSBuildProjectReferences(project):
     references = []
     if project.dependencies:
         group = ["ItemGroup"]
+        added_dependency_set = set()
         for dependency in project.dependencies:
+            dependency_spec = dependency.spec
+            should_skip_dep = False
+            if project.spec["toolset"] == "target":
+                if dependency_spec["toolset"] == "host":
+                    if dependency_spec["type"] == "static_library":
+                        should_skip_dep = True
+            if dependency.name.startswith("run_"):
+                should_skip_dep = False
+            if should_skip_dep:
+                continue
+
+            canonical_name = dependency.name.replace("_host", "")
+            added_dependency_set.add(canonical_name)
             guid = dependency.guid
             project_dir = os.path.split(project.path)[0]
             relative_path = gyp.common.RelativePath(dependency.path, project_dir)
@@ -3675,7 +3701,7 @@ def _GetMSBuildProjectReferences(project):
     return references
 
 
-def _GenerateMSBuildProject(project, options, version, generator_flags):
+def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
     spec = project.spec
     configurations = spec["configurations"]
     project_dir, project_file_name = os.path.split(project.path)
@@ -3774,7 +3800,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
         },
     ]
 
-    content += _GetMSBuildProjectConfigurations(configurations)
+    content += _GetMSBuildProjectConfigurations(configurations, spec)
     content += _GetMSBuildGlobalProperties(
         spec, version, project.guid, project_file_name
     )
@@ -3789,7 +3815,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
     if spec.get("msvs_enable_marmasm"):
         content += import_marmasm_props_section
     content += _GetMSBuildExtensions(props_files_of_rules)
-    content += _GetMSBuildPropertySheets(configurations)
+    content += _GetMSBuildPropertySheets(configurations, spec)
     content += macro_section
     content += _GetMSBuildConfigurationGlobalProperties(
         spec, configurations, project.build_file
@@ -3893,15 +3919,27 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
     sources_handled_by_action = OrderedSet()
     actions_spec = []
     for primary_input, actions in actions_to_add.items():
+        if generator_supports_multiple_toolsets:
+            primary_input = primary_input.replace(".exe", "_host.exe")
         inputs = OrderedSet()
         outputs = OrderedSet()
         descriptions = []
         commands = []
         for action in actions:
+
+            def fixup_host_exe(i):
+                if "$(OutDir)" in i:
+                    i = i.replace(".exe", "_host.exe")
+                return i
+
+            if generator_supports_multiple_toolsets:
+                action["inputs"] = [fixup_host_exe(i) for i in action["inputs"]]
             inputs.update(OrderedSet(action["inputs"]))
             outputs.update(OrderedSet(action["outputs"]))
             descriptions.append(action["description"])
             cmd = action["command"]
+            if generator_supports_multiple_toolsets:
+                cmd = cmd.replace(".exe", "_host.exe")
             # For most actions, add 'call' so that actions that invoke batch files
             # return and continue executing.  msbuild_use_call provides a way to
             # disable this but I have not seen any adverse effect from doing that