Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: support apple silicon (arm64 darwin) builds #2165

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions gyp/.github/workflows/Python_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ jobs:
os: [macos-latest, ubuntu-latest] # , windows-latest]
python-version: [2.7, 3.6, 3.7, 3.8] # 3.5,
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
Expand Down
41 changes: 41 additions & 0 deletions gyp/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Changelog

All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## [Unreleased]

## [0.4.0] - 2020-07-14

### Added
- Added support for passing arbitrary architectures to Xcode builds, enables `arm64` builds.

### Fixed
- Fixed a bug on Solaris where copying archives failed.

## [0.3.0] - 2020-06-06

### Added
- Added support for MSVC cross-compilation. This allows compilation on x64 for
a Windows ARM target.

### Fixed
- Fixed XCode CLT version detection on macOS Catalina.

## [0.2.1] - 2020-05-05

### Fixed
- Relicensed to Node.js contributors.
- Fixed Windows bug introduced in v0.2.0.

## [0.2.0] - 2020-04-06

This is the first release of this project, based on https://chromium.googlesource.com/external/gyp
with changes made over the years in Node.js and node-gyp.

[Unreleased]: https://github.com/nodejs/gyp-next/compare/v0.4.0...HEAD
[0.4.0]: https://github.com/nodejs/gyp-next/compare/v0.3.0...v0.4.0
[0.3.0]: https://github.com/nodejs/gyp-next/compare/v0.2.1...v0.3.0
[0.2.1]: https://github.com/nodejs/gyp-next/compare/v0.2.0...v0.2.1
[0.2.0]: https://github.com/nodejs/gyp-next/releases/tag/v0.2.0
4 changes: 2 additions & 2 deletions gyp/pylib/gyp/generator/android.py
Original file line number Diff line number Diff line change
Expand Up @@ -981,9 +981,9 @@ def WriteList(
"""
values = ""
if value_list:
value_list = [quoter(prefix + l) for l in value_list]
value_list = [quoter(prefix + value) for value in value_list]
if local_pathify:
value_list = [self.LocalPathify(l) for l in value_list]
value_list = [self.LocalPathify(value) for value in value_list]
values = " \\\n\t" + " \\\n\t".join(value_list)
self.fp.write("%s :=%s\n\n" % (variable, values))

Expand Down
14 changes: 11 additions & 3 deletions gyp/pylib/gyp/generator/make.py
Original file line number Diff line number Diff line change
Expand Up @@ -1342,7 +1342,7 @@ def WriteSources(
)

if self.flavor == "mac":
cflags = self.xcode_settings.GetCflags(configname)
cflags = self.xcode_settings.GetCflags(configname, arch=config.get('xcode_configuration_platform'))
cflags_c = self.xcode_settings.GetCflagsC(configname)
cflags_cc = self.xcode_settings.GetCflagsCC(configname)
cflags_objc = self.xcode_settings.GetCflagsObjC(configname)
Expand Down Expand Up @@ -1637,6 +1637,7 @@ def WriteTarget(
configname,
generator_default_variables["PRODUCT_DIR"],
lambda p: Sourceify(self.Absolutify(p)),
arch=config.get('xcode_configuration_platform')
)

# TARGET_POSTBUILDS_$(BUILDTYPE) is added to postbuilds later on.
Expand Down Expand Up @@ -1944,7 +1945,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar
"""
values = ""
if value_list:
value_list = [quoter(prefix + l) for l in value_list]
value_list = [quoter(prefix + value) for value in value_list]
values = " \\\n\t" + " \\\n\t".join(value_list)
self.fp.write("%s :=%s\n\n" % (variable, values))

Expand Down Expand Up @@ -2362,7 +2363,14 @@ def CalculateMakefilePath(build_file, base_name):
}
)
elif flavor == "solaris":
header_params.update({"flock": "./gyp-flock-tool flock", "flock_index": 2})
copy_archive_arguments = "-pPRf@"
header_params.update(
{
"copy_archive_args": copy_archive_arguments,
"flock": "./gyp-flock-tool flock",
"flock_index": 2
}
)
elif flavor == "freebsd":
# Note: OpenBSD has sysutils/flock. lockf seems to be FreeBSD specific.
header_params.update({"flock": "lockf"})
Expand Down
96 changes: 67 additions & 29 deletions gyp/pylib/gyp/generator/msvs.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
# letters.
VALID_MSVS_GUID_CHARS = re.compile(r"^[A-F0-9\-]+$")

generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()

generator_default_variables = {
"DRIVER_PREFIX": "",
Expand All @@ -50,7 +51,7 @@
"STATIC_LIB_SUFFIX": ".lib",
"SHARED_LIB_SUFFIX": ".dll",
"INTERMEDIATE_DIR": "$(IntDir)",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)obj/global_intermediate",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)/obj/global_intermediate",
"OS": "win",
"PRODUCT_DIR": "$(OutDir)",
"LIB_DIR": "$(OutDir)lib",
Expand Down Expand Up @@ -1005,7 +1006,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
return toolset


def _GenerateProject(project, options, version, generator_flags):
def _GenerateProject(project, options, version, generator_flags, spec):
"""Generates a vcproj file.

Arguments:
Expand All @@ -1023,7 +1024,7 @@ def _GenerateProject(project, options, version, generator_flags):
return []

if version.UsesVcxproj():
return _GenerateMSBuildProject(project, options, version, generator_flags)
return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
else:
return _GenerateMSVSProject(project, options, version, generator_flags)

Expand Down Expand Up @@ -1903,6 +1904,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
# Convert into a tree of dicts on path.
for p in sln_projects:
gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
if p.endswith("#host"):
target += "_host"
gyp_dir = os.path.dirname(gyp_file)
path_dict = _GetPathDict(root, gyp_dir)
path_dict[target + ".vcproj"] = project_objects[p]
Expand All @@ -1921,9 +1924,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
default_config = _GetDefaultConfiguration(spec)
proj_filename = default_config.get("msvs_existing_vcproj")
if not proj_filename:
proj_filename = (
spec["target_name"] + options.suffix + msvs_version.ProjectExtension()
)
proj_filename = spec["target_name"]
if spec["toolset"] == "host":
proj_filename += "_host"
proj_filename = proj_filename + options.suffix + msvs_version.ProjectExtension()

build_file = gyp.common.BuildFile(qualified_target)
proj_path = os.path.join(os.path.dirname(build_file), proj_filename)
Expand All @@ -1948,6 +1952,8 @@ def _GetPlatformOverridesOfProject(spec):
_ConfigBaseName(config_name, _ConfigPlatform(c)),
platform,
)
if spec["toolset"] == "host" and generator_supports_multiple_toolsets:
fixed_config_fullname = "%s|x64" % (config_name,)
config_platform_overrides[config_fullname] = fixed_config_fullname
return config_platform_overrides

Expand All @@ -1968,21 +1974,19 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
projects = {}
for qualified_target in target_list:
spec = target_dicts[qualified_target]
if spec["toolset"] != "target":
raise GypError(
"Multiple toolsets not supported in msvs build (target %s)"
% qualified_target
)
proj_path, fixpath_prefix = _GetPathOfProject(
qualified_target, spec, options, msvs_version
)
guid = _GetGuidOfProject(proj_path, spec)
overrides = _GetPlatformOverridesOfProject(spec)
build_file = gyp.common.BuildFile(qualified_target)
# Create object for this project.
target_name = spec["target_name"]
if spec["toolset"] == "host":
target_name += "_host"
obj = MSVSNew.MSVSProject(
proj_path,
name=spec["target_name"],
name=target_name,
guid=guid,
spec=spec,
build_file=build_file,
Expand Down Expand Up @@ -2161,7 +2165,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
for qualified_target in target_list:
spec = target_dicts[qualified_target]
for config_name, config in spec["configurations"].items():
configs.add(_ConfigFullName(config_name, config))
config_name = _ConfigFullName(config_name, config)
configs.add(config_name)
if config_name == "Release|arm64":
configs.add("Release|x64")
configs = list(configs)

# Figure out all the projects that will be generated and their guids
Expand All @@ -2174,12 +2181,15 @@ def GenerateOutput(target_list, target_dicts, data, params):
for project in project_objects.values():
fixpath_prefix = project.fixpath_prefix
missing_sources.extend(
_GenerateProject(project, options, msvs_version, generator_flags)
_GenerateProject(project, options, msvs_version, generator_flags, spec)
)
fixpath_prefix = None

for build_file in data:
# Validate build_file extension
target_only_configs = configs
if generator_supports_multiple_toolsets:
target_only_configs = [i for i in configs if i.endswith("arm64")]
if not build_file.endswith(".gyp"):
continue
sln_path = os.path.splitext(build_file)[0] + options.suffix + ".sln"
Expand All @@ -2196,7 +2206,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
sln = MSVSNew.MSVSSolution(
sln_path,
entries=root_entries,
variants=configs,
variants=target_only_configs,
websiteProperties=False,
version=msvs_version,
)
Expand Down Expand Up @@ -2930,22 +2940,24 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)


def _GetConfigurationAndPlatform(name, settings):
def _GetConfigurationAndPlatform(name, settings, spec):
configuration = name.rsplit("_", 1)[0]
platform = settings.get("msvs_configuration_platform", "Win32")
if spec["toolset"] == "host" and platform == "arm64":
platform = "x64" # Host-only tools are always built for x64
return (configuration, platform)


def _GetConfigurationCondition(name, settings):
def _GetConfigurationCondition(name, settings, spec):
return r"'$(Configuration)|$(Platform)'=='%s|%s'" % _GetConfigurationAndPlatform(
name, settings
name, settings, spec
)


def _GetMSBuildProjectConfigurations(configurations):
def _GetMSBuildProjectConfigurations(configurations, spec):
group = ["ItemGroup", {"Label": "ProjectConfigurations"}]
for (name, settings) in sorted(configurations.items()):
configuration, platform = _GetConfigurationAndPlatform(name, settings)
configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
designation = "%s|%s" % (configuration, platform)
group.append(
[
Expand Down Expand Up @@ -3033,7 +3045,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
properties = {}
for name, settings in spec["configurations"].items():
msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
condition = _GetConfigurationCondition(name, settings)
condition = _GetConfigurationCondition(name, settings, spec)
character_set = msbuild_attributes.get("CharacterSet")
config_type = msbuild_attributes.get("ConfigurationType")
_AddConditionalProperty(properties, condition, "ConfigurationType", config_type)
Expand Down Expand Up @@ -3064,12 +3076,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
return properties


def _GetMSBuildPropertySheets(configurations):
def _GetMSBuildPropertySheets(configurations, spec):
user_props = r"$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
additional_props = {}
props_specified = False
for name, settings in sorted(configurations.items()):
configuration = _GetConfigurationCondition(name, settings)
configuration = _GetConfigurationCondition(name, settings, spec)
if "msbuild_props" in settings:
additional_props[configuration] = _FixPaths(settings["msbuild_props"])
props_specified = True
Expand Down Expand Up @@ -3222,7 +3234,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):

properties = {}
for (name, configuration) in sorted(configurations.items()):
condition = _GetConfigurationCondition(name, configuration)
condition = _GetConfigurationCondition(name, configuration, spec)
attributes = _GetMSBuildAttributes(spec, configuration, build_file)
msbuild_settings = configuration["finalized_msbuild_settings"]
_AddConditionalProperty(
Expand Down Expand Up @@ -3345,7 +3357,7 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
msbuild_settings = configuration["finalized_msbuild_settings"]
group = [
"ItemDefinitionGroup",
{"Condition": _GetConfigurationCondition(name, configuration)},
{"Condition": _GetConfigurationCondition(name, configuration, spec)},
]
for tool_name, tool_settings in sorted(msbuild_settings.items()):
# Skip the tool named '' which is a holder of global settings handled
Expand Down Expand Up @@ -3625,7 +3637,7 @@ def _AddSources2(

if precompiled_source == source:
condition = _GetConfigurationCondition(
config_name, configuration
config_name, configuration, spec
)
detail.append(
["PrecompiledHeader", {"Condition": condition}, "Create"]
Expand All @@ -3652,7 +3664,21 @@ def _GetMSBuildProjectReferences(project):
references = []
if project.dependencies:
group = ["ItemGroup"]
added_dependency_set = set()
for dependency in project.dependencies:
dependency_spec = dependency.spec
should_skip_dep = False
if project.spec["toolset"] == "target":
if dependency_spec["toolset"] == "host":
if dependency_spec["type"] == "static_library":
should_skip_dep = True
if dependency.name.startswith("run_"):
should_skip_dep = False
if should_skip_dep:
continue

canonical_name = dependency.name.replace("_host", "")
added_dependency_set.add(canonical_name)
guid = dependency.guid
project_dir = os.path.split(project.path)[0]
relative_path = gyp.common.RelativePath(dependency.path, project_dir)
Expand All @@ -3675,7 +3701,7 @@ def _GetMSBuildProjectReferences(project):
return references


def _GenerateMSBuildProject(project, options, version, generator_flags):
def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
spec = project.spec
configurations = spec["configurations"]
project_dir, project_file_name = os.path.split(project.path)
Expand Down Expand Up @@ -3774,7 +3800,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
},
]

content += _GetMSBuildProjectConfigurations(configurations)
content += _GetMSBuildProjectConfigurations(configurations, spec)
content += _GetMSBuildGlobalProperties(
spec, version, project.guid, project_file_name
)
Expand All @@ -3789,7 +3815,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
if spec.get("msvs_enable_marmasm"):
content += import_marmasm_props_section
content += _GetMSBuildExtensions(props_files_of_rules)
content += _GetMSBuildPropertySheets(configurations)
content += _GetMSBuildPropertySheets(configurations, spec)
content += macro_section
content += _GetMSBuildConfigurationGlobalProperties(
spec, configurations, project.build_file
Expand Down Expand Up @@ -3893,15 +3919,27 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
sources_handled_by_action = OrderedSet()
actions_spec = []
for primary_input, actions in actions_to_add.items():
if generator_supports_multiple_toolsets:
primary_input = primary_input.replace(".exe", "_host.exe")
inputs = OrderedSet()
outputs = OrderedSet()
descriptions = []
commands = []
for action in actions:

def fixup_host_exe(i):
if "$(OutDir)" in i:
i = i.replace(".exe", "_host.exe")
return i

if generator_supports_multiple_toolsets:
action["inputs"] = [fixup_host_exe(i) for i in action["inputs"]]
inputs.update(OrderedSet(action["inputs"]))
outputs.update(OrderedSet(action["outputs"]))
descriptions.append(action["description"])
cmd = action["command"]
if generator_supports_multiple_toolsets:
cmd = cmd.replace(".exe", "_host.exe")
# For most actions, add 'call' so that actions that invoke batch files
# return and continue executing. msbuild_use_call provides a way to
# disable this but I have not seen any adverse effect from doing that
Expand Down
Loading