Skip to content

Build infra changes to enable building a more universal cross target jit builds #41126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
Sep 9, 2020
Merged

Build infra changes to enable building a more universal cross target jit builds #41126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
906d468
Code changes needed to compile targetting all supported os/arch from …
davidwrighton Aug 17, 2020
83ed473
Fix unix to windows cross target build issues
davidwrighton Aug 20, 2020
482e8c3
Build rules to build all possible altjits
davidwrighton Aug 18, 2020
824419d
New build subset rules
davidwrighton Aug 19, 2020
a56cc15
Adjust naming to clrjit_targetos_targetarch_hostarch
davidwrighton Aug 20, 2020
3d359df
Fix subsets build so that clr subset compiles properly
davidwrighton Aug 20, 2020
d471312
Swap crossgen2 to use new clrjit dlls
davidwrighton Aug 21, 2020
1afd7e1
Use target defintions and properties to simplify logic
davidwrighton Aug 21, 2020
72dd26d
Stop using subdirectories for cross targeting jits and such
davidwrighton Aug 21, 2020
ef86678
Consolidate all standalone jit building to use the function instead o…
davidwrighton Aug 22, 2020
7d3c823
Fix build break, and apply formatting patch
davidwrighton Aug 24, 2020
db74ff8
Address code review feedback, and fix pgo build and armel jit build
davidwrighton Aug 24, 2020
d7a5a91
Fix Windows Arm build, and don't compile clrjit as altjit, ever
davidwrighton Aug 25, 2020
8b184d7
Fix crossgen2 packaging on Unix
davidwrighton Aug 25, 2020
fa08b56
Code review feedback
davidwrighton Aug 26, 2020
0f420f6
Make jitinterface be host specific, and fix build of windows crosstar…
davidwrighton Sep 2, 2020
bbeceaf
Fix issue blocking compilation of X86 targetted code
davidwrighton Sep 2, 2020
b0703f7
Improve R2R dump
davidwrighton Sep 2, 2020
f5e1064
Fix difference in x86 r2r image generation between x64 and x86 hosted
davidwrighton Sep 2, 2020
c2a2789
Address code review feedback
davidwrighton Sep 3, 2020
42b44da
Fix issues found in CI
davidwrighton Sep 4, 2020
92fe566
Fix formatting
davidwrighton Sep 4, 2020
8da95fb
add comment as requested
davidwrighton Sep 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions eng/Signing.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,15 @@
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.DependencyAnalysisFramework.dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.ReadyToRun.dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)ILCompiler.TypeSystem.ReadyToRun.dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)jitinterface.dll" />

<ItemsToSign Condition="'$(TargetOS)' == 'Windows_NT'" Include="$(CoreCLRCrossgen2Dir)clrjit-win-$(TargetArchitecture).dll" />
<ItemsToSign Condition="'$(TargetOS)' != 'Windows_NT'" Include="$(CoreCLRCrossgen2Dir)clrjit-unix-$(TargetArchitecture).dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)jitinterface_$(TargetArchitecture).dll" />

<ItemsToSign Include="$(CoreCLRCrossgen2Dir)clrjit_win_x86_$(TargetArchitecture).dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)clrjit_win_arm_$(TargetArchitecture).dll" />
<ItemsToSign Include="$(CoreCLRCrossgen2Dir)clrjit_unix_arm_$(TargetArchitecture).dll" />
<ItemsToSign Condition="'$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'x64'" Include="$(CoreCLRCrossgen2Dir)clrjit_win_x64_$(TargetArchitecture).dll" />
<ItemsToSign Condition="'$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'x64'" Include="$(CoreCLRCrossgen2Dir)clrjit_win_arm64_$(TargetArchitecture).dll" />
<ItemsToSign Condition="'$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'x64'" Include="$(CoreCLRCrossgen2Dir)clrjit_unix_x64_$(TargetArchitecture).dll" />
<ItemsToSign Condition="'$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'x64'" Include="$(CoreCLRCrossgen2Dir)clrjit_unix_arm64_$(TargetArchitecture).dll" />

<ItemsToSign Include="@(CoreCLRCrossTargetItemsToSign)" />

Expand Down
27 changes: 22 additions & 5 deletions eng/Subsets.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
</PropertyGroup>

<PropertyGroup>
<DefaultCoreClrSubsets>clr.runtime+linuxdac+clr.corelib+clr.nativecorelib+clr.tools+clr.packages</DefaultCoreClrSubsets>
<DefaultCoreClrSubsets>clr.runtime+clr.jit+clr.alljits+linuxdac+clr.corelib+clr.nativecorelib+clr.tools+clr.packages</DefaultCoreClrSubsets>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't "clr.alljits" contain "clr.jit", so only "clr.alljits" is necessary?


<DefaultMonoSubsets Condition="'$(MonoEnableLLVM)' == 'true' and '$(MonoLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
<DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets>
Expand All @@ -74,12 +74,17 @@

<!-- Surround _subset in dashes to simplify checks below -->
<_subset>+$(_subset.Trim('+'))+</_subset>

<ClrRuntimeBuildSubsets></ClrRuntimeBuildSubsets>
<ClrDefaultRuntimeBuildSubsets>ClrRuntimeSubset=true;ClrJitSubset=true</ClrDefaultRuntimeBuildSubsets>
</PropertyGroup>

<ItemGroup>
<!-- CoreClr -->
<SubsetName Include="Clr" Description="The CoreCLR runtime, LinuxDac, CoreLib (+ native), tools and packages." />
<SubsetName Include="Clr.Runtime" Description="The CoreCLR .NET runtime." />
<SubsetName Include="Clr.Jit" Description="The JIT for the CoreCLR .NET runtime." />
<SubsetName Include="Clr.AllJits" Description="All of the cross-targeting JIT compilers for the CoreCLR .NET runtime." />
<SubsetName Include="Clr.CoreLib" Description="The managed System.Private.CoreLib library for CoreCLR." />
<SubsetName Include="Clr.NativeCoreLib" Description="Run crossgen on System.Private.CoreLib library for CoreCLR." />
<SubsetName Include="Clr.Tools" Description="Managed tools that support CoreCLR development and testing." />
Expand Down Expand Up @@ -132,16 +137,28 @@
<ProjectToBuild Include="$(CoreClrProjectRoot)src\System.Private.CoreLib\System.Private.CoreLib.csproj" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+clr.runtime+'))">
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" Category="clr" />
<PropertyGroup Condition="$(_subset.Contains('+clr.runtime+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrRuntimeSubset=true;ClrJitSubset=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.jit+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrJitSubset=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.alljits+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrAllJitsSubset=true;ClrJitSubset=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<ItemGroup Condition="'$(ClrRuntimeBuildSubsets)' != ''">
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);$(ClrRuntimeBuildSubsets)" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+linuxdac+')) and $([MSBuild]::IsOsPlatform(Windows)) and '$(TargetArchitecture)' != 'x86'">
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);CrossDac=linux" Category="clr" />
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);$(ClrDefaultRuntimeBuildSubsets);CrossDac=linux" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+alpinedac+')) and $([MSBuild]::IsOsPlatform(Windows)) and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'arm'">
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);CrossDac=alpine" Category="clr" />
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);$(ClrDefaultRuntimeBuildSubsets);CrossDac=alpine" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+crossdacpack+'))">
Expand Down
36 changes: 22 additions & 14 deletions eng/native/configurecompiler.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,13 +201,17 @@ endif()
# Definitions (for platform)
#-----------------------------------
if (CLR_CMAKE_HOST_ARCH_AMD64)
set(ARCH_HOST_NAME x64)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure of the naming conventions, but should these be CLR_ARCH_HOST_NAME? Maybe even CLR_ARCH_HOST_DISPLAY_NAME to emphasize that this name will be displayed/visible to users?

Same applies to ARCH_TARGET_NAME

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would not prefix it with CLR_. This prefix is used when the variable would start with CMAKE_ to distinguish it from the CMake's ones.

add_definitions(-DHOST_AMD64)
add_definitions(-DHOST_64BIT)
elseif (CLR_CMAKE_HOST_ARCH_I386)
set(ARCH_HOST_NAME x86)
add_definitions(-DHOST_X86)
elseif (CLR_CMAKE_HOST_ARCH_ARM)
set(ARCH_HOST_NAME arm)
add_definitions(-DHOST_ARM)
elseif (CLR_CMAKE_HOST_ARCH_ARM64)
set(ARCH_HOST_NAME arm64)
add_definitions(-DHOST_ARM64)
add_definitions(-DHOST_64BIT)
else ()
Expand Down Expand Up @@ -262,18 +266,22 @@ endif(CLR_CMAKE_HOST_WIN32)
# Architecture specific files folder name
if (CLR_CMAKE_TARGET_ARCH_AMD64)
set(ARCH_SOURCES_DIR amd64)
add_definitions(-DTARGET_AMD64)
add_definitions(-DTARGET_64BIT)
set(ARCH_TARGET_NAME x64)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_AMD64>)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's some cmake magic here with IGNORE_DEFAULT_TARGET_ARCH that I don't understand (I've seen the same magic in the JIT CMakeLists.txt). Maybe a comment here (and there) would be helpful?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This just means - add the definition only if the target being compiled doesn't have the IGNORE_DEFAULT_TARGET_ARCH property set.
But I am not happy with this way of solving the actual problem. This file is meant to define general properties for all targets being built in the runtime repo (in coreclr, libs, installer) and possibly other repos that include this file (I think e.g. diagnostics repo does that too). What we do here is special to coreclr. It also seems error prone - it will be easy to forget / hard to figure out when adding a new define in the future where to use this conditional adding of target property and when to not to use it and we now have the same defines created at two places (here and in the src/coreclr/clrdefinitions.cmake for the case we use the IGNORE_DEFAULT_TARGET_ARCH property.
Since we already had a build of some cross components (including e.g. x64 hosted jit for x86 / arm) before your change, so I wonder why we cannot use the same mechanism using the TARGET_* / HOST_* defines.

add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_64BIT>)
elseif (CLR_CMAKE_TARGET_ARCH_ARM64)
set(ARCH_SOURCES_DIR arm64)
add_definitions(-DTARGET_ARM64)
add_definitions(-DTARGET_64BIT)
set(ARCH_TARGET_NAME arm64)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_ARM64>)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_64BIT>)
elseif (CLR_CMAKE_TARGET_ARCH_ARM)
set(ARCH_SOURCES_DIR arm)
add_definitions(-DTARGET_ARM)
set(ARCH_TARGET_NAME arm)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_ARM>)
elseif (CLR_CMAKE_TARGET_ARCH_I386)
set(ARCH_TARGET_NAME x86)
set(ARCH_SOURCES_DIR i386)
add_definitions(-DTARGET_X86)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:TARGET_X86>)
else ()
clr_unknown_arch()
endif ()
Expand Down Expand Up @@ -374,24 +382,24 @@ if (CLR_CMAKE_HOST_UNIX)
endif(CLR_CMAKE_HOST_UNIX)

if(CLR_CMAKE_TARGET_UNIX)
add_definitions(-DTARGET_UNIX)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_UNIX>)
# Contracts are disabled on UNIX.
add_definitions(-DDISABLE_CONTRACTS)
if(CLR_CMAKE_TARGET_OSX)
add_definitions(-DTARGET_OSX)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_OSX>)
elseif(CLR_CMAKE_TARGET_FREEBSD)
add_definitions(-DTARGET_FREEBSD)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_FREEBSD>)
elseif(CLR_CMAKE_TARGET_LINUX)
add_definitions(-DTARGET_LINUX)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_LINUX>)
elseif(CLR_CMAKE_TARGET_NETBSD)
add_definitions(-DTARGET_NETBSD)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_NETBSD>)
elseif(CLR_CMAKE_TARGET_SUNOS)
add_definitions(-DTARGET_SUNOS)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_SUNOS>)
elseif(CLR_CMAKE_TARGET_ANDROID)
add_definitions(-DTARGET_ANDROID)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_ANDROID>)
endif()
else(CLR_CMAKE_TARGET_UNIX)
add_definitions(-DTARGET_WINDOWS)
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_OS>>>:TARGET_WINDOWS>)
endif(CLR_CMAKE_TARGET_UNIX)

if(CLR_CMAKE_HOST_UNIX_ARM)
Expand Down
4 changes: 2 additions & 2 deletions eng/native/configureplatform.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,8 +364,8 @@ endif(CLR_CMAKE_TARGET_UNIX)
# check if host & target os/arch combination are valid
if (CLR_CMAKE_TARGET_OS STREQUAL CLR_CMAKE_HOST_OS)
if(NOT(CLR_CMAKE_TARGET_ARCH STREQUAL CLR_CMAKE_HOST_ARCH))
if(NOT((CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM64) OR (CLR_CMAKE_HOST_ARCH_I386 AND CLR_CMAKE_TARGET_ARCH_ARM) OR (CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM)))
message(FATAL_ERROR "Invalid platform and target arch combination")
if(NOT((CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM64) OR (CLR_CMAKE_HOST_ARCH_I386 AND CLR_CMAKE_TARGET_ARCH_ARM) OR (CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_ARM) OR (CLR_CMAKE_HOST_ARCH_AMD64 AND CLR_CMAKE_TARGET_ARCH_I386)))
message(FATAL_ERROR "Invalid platform and target arch combination TARGET_ARCH=${CLR_CMAKE_TARGET_ARCH} HOST_ARCH=${CLR_CMAKE_HOST_ARCH}")
endif()
endif()
else()
Expand Down
10 changes: 6 additions & 4 deletions src/coreclr/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ include(pgosupport.cmake)
#-------------------------------
# Include libraries native shims
#-------------------------------
if(NOT CLR_CROSS_COMPONENTS_BUILD)
if(NOT CLR_CROSS_COMPONENTS_BUILD AND CLR_CMAKE_BUILD_SUBSET_RUNTIME)
add_subdirectory(src/libraries-native)
endif(NOT CLR_CROSS_COMPONENTS_BUILD)
endif(NOT CLR_CROSS_COMPONENTS_BUILD AND CLR_CMAKE_BUILD_SUBSET_RUNTIME)

#-----------------------------------------
# Add Projects
Expand Down Expand Up @@ -86,7 +86,7 @@ add_subdirectory(src/pal/prebuilt/inc)

add_subdirectory(src/debug/debug-pal)

if(CLR_CMAKE_TARGET_WIN32)
if(CLR_CMAKE_TARGET_WIN32 AND CLR_CMAKE_BUILD_SUBSET_RUNTIME)
add_subdirectory(src/gc/sample)
endif()

Expand Down Expand Up @@ -115,7 +115,9 @@ include_directories("../../artifacts/obj/coreclr")

if(FEATURE_STANDALONE_GC)
add_definitions(-DFEATURE_STANDALONE_GC)
add_subdirectory(src/gc)
if (CLR_CMAKE_BUILD_SUBSET_RUNTIME)
add_subdirectory(src/gc)
endif (CLR_CMAKE_BUILD_SUBSET_RUNTIME)
endif(FEATURE_STANDALONE_GC)

if (CLR_CMAKE_HOST_UNIX)
Expand Down
Loading