CoreCLR runtime tests + Mono LLVM AOT on arm64 Linux (#41751)

This change:

- Adds new options to mono.proj:

    - MonoAOTEnableLLVM, which enables (or disables) building a Mono AOT
    cross compiler with LLVM;

    - MonoAOTLLVMDir, which specifies the path to a copy of LLVM suitable for
    building a Mono AOT cross compiler, and is optional; and

    - BuildMonoAOTCrossCompilerOnly, which allows building a Mono AOT
    cross compiler without building an associated full Mono runtime.

- Changes offsets-tool.py's user interface slightly; '--include-prefix' is
renamed to '--prefix' and may be specified multiple times. While this latter
feature isn't necessary to build a Mono cross compiler today, because we don't
use distribution-supplied cross-compilation headers on CI, it does make it
easier to experiment with the offsets tool using arbitrary header layouts.  For
example, Debian's arm64 cross-compilation packages scatter useful header files
across `/usr/lib/gcc-cross/aarch64-linux-gnu` and `/usr/aarch64-linux-gnu`.

- Updates the docker image used for arm64 cross-compilation to a newer revision
that includes libclang (see also
dotnet/dotnet-buildtools-prereqs-docker#375).

- Adds some tests that currently fail to compile with Mono LLVM AOT to
issues.targets.

- Adds a Linux_arm64 LLVM AOT job to CI. Nothing particularly fancy is done to
build the Mono LLVM AOT cross compiler; it is built as a step in the same job
that also sends Linux_arm64 tests to Helix.

Author: imhameed

docs/workflow/building/coreclr/linux-instructions.md

@@ -53,7 +53,7 @@ This table of images might often become stale as we change our images as our req
 | Alpine                      | x64             | `mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015`             | -                    | -clang9       |
 | CentOS 7 (build for RHEL 7) | x64             | `mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-359e48e-20200313130914`                        | -                    | -clang9       |
 | Ubuntu 16.04                | arm32 (armhf)   | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-20200413125008-09ec757`              | `/crossrootfs/arm`   | -clang9       |
-| Ubuntu 16.04                | arm64 (arm64v8) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-20200413125008-cfdd435`        | `/crossrootfs/arm64` | -clang9       |
+| Ubuntu 16.04                | arm64 (arm64v8) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-20201022204150-b2c2436`        | `/crossrootfs/arm64` | -clang9       |
 | Alpine                      | arm64 (arm64v8) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-20200413125008-406629a` | `/crossrootfs/arm64` | -clang5.0     |
 
 Environment

eng/Subsets.props

@@ -57,6 +57,7 @@
     <DefaultCoreClrSubsets>clr.runtime+clr.jit+clr.alljits+linuxdac+clr.corelib+clr.nativecorelib+clr.tools+clr.packages</DefaultCoreClrSubsets>
 
     <DefaultMonoSubsets Condition="'$(MonoEnableLLVM)' == 'true' and '$(MonoLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
+    <DefaultMonoSubsets Condition="'$(MonoAOTEnableLLVM)' == 'true' and '$(MonoAOTLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
     <DefaultMonoSubsets Condition="'$(TargetOS)' == 'Browser'">$(DefaultMonoSubsets)mono.wasmruntime+</DefaultMonoSubsets>
     <DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets>
 

eng/pipelines/common/platform-matrix.yml

@@ -64,7 +64,7 @@ jobs:
       targetRid: linux-arm64
       platform: Linux_arm64
       container:
-        image: ubuntu-16.04-cross-arm64-20200413125008-cfdd435
+        image: ubuntu-16.04-cross-arm64-20201022204150-b2c2436
         registry: mcr
       jobParameters:
         runtimeFlavor: ${{ parameters.runtimeFlavor }}

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -223,7 +223,6 @@ jobs:
           artifactName: '$(coreClrProductArtifactName)'
           displayName: 'CoreCLR product download for Mono'
 
-
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing
     # ilproj test projects during copynativeonly.
     - template: /eng/pipelines/common/download-artifact-step.yml
@@ -263,6 +262,20 @@ jobs:
     - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) generatelayoutonly $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
       displayName: Generate CORE_ROOT
 
+    # Build a Mono LLVM AOT cross-compiler for non-amd64 targets (in this case, just arm64)
+    - ${{ if and(eq(parameters.runtimeFlavor, 'mono'), eq(parameters.runtimeVariant, 'llvmaot')) }}:
+      - ${{ if eq(parameters.archType, 'arm64') }}:
+        - script: ./build.sh
+                  -subset mono
+                  -c $(buildConfigUpper)
+                  -arch $(archType)
+                  /p:BuildMonoAotCrossCompiler=true
+                  /p:BuildMonoAotCrossCompilerOnly=true
+                  /p:MonoLibClang="/usr/lib/llvm-9/lib/libclang-9.so.1"
+                  /p:MonoAOTEnableLLVM=true
+                  /p:MonoAOTLLVMUseCxx11Abi=true
+          displayName: "Build Mono LLVM AOT cross compiler"
+
 
     # Overwrite coreclr runtime binaries with mono ones
     - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
@@ -274,8 +287,14 @@ jobs:
         displayName: "Patch dotnet with mono"
 
     - ${{ if and(eq(parameters.runtimeFlavor, 'mono'), eq(parameters.runtimeVariant, 'llvmaot')) }}:
-      - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig)
-        displayName: "LLVM AOT compile CoreCLR tests"
+      - ${{ if eq(parameters.archType, 'x64') }}:
+        - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType)
+          displayName: "LLVM AOT compile CoreCLR tests"
+      - ${{ if eq(parameters.archType, 'arm64') }}:
+        - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType) cross
+          displayName: "LLVM AOT cross-compile CoreCLR tests"
+          env:
+            __MonoToolPrefix: aarch64-linux-gnu-
 
     # Send tests to Helix
     - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

eng/pipelines/mono/templates/build-job.yml

@@ -74,12 +74,15 @@ jobs:
         value: wasm
       - name: osOverride
         value: '-os Browser'
+    - ${{ if and(eq(parameters.osGroup, 'Linux'), not(eq(parameters.archType, 'x64'))) }}:
+        name: llvmCxxAbi
+        value: /p:MonoLLVMUseCxx11Abi=true
     - ${{ if eq(parameters.runtimeVariant, 'llvmjit') }}:
       - name: llvmParameter
-        value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false
+        value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false $(llvmCxxAbi)
     - ${{ if eq(parameters.runtimeVariant, 'llvmaot') }}:
       - name: llvmParameter
-        value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true
+        value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true $(llvmCxxAbi)
     - ${{ parameters.variables }}
 
     steps:
@@ -96,7 +99,7 @@ jobs:
       - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
         displayName: Install native dependencies
 
-    - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
+    - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
       - script: |
           du -sh $(Build.SourcesDirectory)/*
           df -h
@@ -110,7 +113,7 @@ jobs:
       - script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter)
         displayName: Build product
 
-    - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
+    - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
       - script: |
           du -sh $(Build.SourcesDirectory)/*
           df -h

eng/pipelines/runtime.yml

@@ -585,6 +585,7 @@ jobs:
     buildConfig: release
     platforms:
     - Linux_x64
+    - Linux_arm64
     jobParameters:
       runtimeVariant: llvmaot
       condition: >-
@@ -853,6 +854,7 @@ jobs:
     runtimeFlavor: mono
     platforms:
     - Linux_x64
+    - Linux_arm64
     helixQueueGroup: pr
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:

src/mono/CMakeLists.txt

@@ -148,6 +148,9 @@ if(NOT AOT_TARGET_TRIPLE STREQUAL "")
   elseif(AOT_TARGET_TRIPLE STREQUAL "wasm32-unknown-none")
     set(TARGET_SYSTEM_NAME "Emscripten")
     set(TARGET_ARCH "wasm")
+  elseif(AOT_TARGET_TRIPLE STREQUAL "aarch64-linux-gnu")
+    set(TARGET_SYSTEM_NAME "Linux")
+    set(TARGET_ARCH "arm64")
   else()
     message(FATAL_ERROR "AOT target '${AOT_TARGET_TRIPLE}' not supported.")
   endif()

src/mono/Directory.Build.props

@@ -3,7 +3,7 @@
     <SkipImportArcadeSdkFromRoot>true</SkipImportArcadeSdkFromRoot>
   </PropertyGroup>
   <Import Project="..\..\Directory.Build.props" />
-  
+
   <!-- Set default Platform -->
   <PropertyGroup>
     <HostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant)</HostArch>
@@ -101,7 +101,8 @@
 
   <PropertyGroup>
     <MonoObjDir>$(ArtifactsObjDir)mono/$(PlatformConfigPathPart)/</MonoObjDir>
-    <MonoLLVMDir Condition="('$(MonoEnableLLVM)' == 'true' or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'Browser') and '$(MonoLLVMDir)' == ''">$(MonoObjDir)llvm</MonoLLVMDir>
+    <MonoLLVMDir Condition="'$(MonoLLVMDir)' == ''">$(MonoObjDir)llvm</MonoLLVMDir>
+    <MonoAOTLLVMDir Condition="'$(MonoAOTLLVMDir)' == ''">$(MonoObjDir)cross/llvm</MonoAOTLLVMDir>
   </PropertyGroup>
 
   <!-- Paths for Mobile App Projects  -->

src/mono/llvm/llvm-init.proj

@@ -25,17 +25,28 @@
   <ItemGroup>
     <PackageReference Include="runtime.$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture).Microsoft.NETCore.Runtime.Mono.LLVM.Sdk" Version="$(MonoLLVMSDKVersion)"/>
     <PackageReference Include="runtime.$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture).Microsoft.NETCore.Runtime.Mono.LLVM.Tools" Version="$(MonoLLVMToolsVersion)"/>
+
+    <PackageReference Include="runtime.$(MonoLLVMHostOS)-$(HostArch).Microsoft.NETCore.Runtime.Mono.LLVM.Sdk" Version="$(MonoLLVMSDKVersion)"/>
+    <PackageReference Include="runtime.$(MonoLLVMHostOS)-$(HostArch).Microsoft.NETCore.Runtime.Mono.LLVM.Tools" Version="$(MonoLLVMToolsVersion)"/>
   </ItemGroup>
 
   <Target Name="CopyLLVMToTree" AfterTargets="Build">
     <ItemGroup>
       <LLVMFiles Include="$(NuGetPackageRoot)\runtime.$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture).microsoft.netcore.runtime.mono.llvm.sdk\$(MonoLLVMSDKVersion)\tools\$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture)\**" />
       <LLVMFiles Include="$(NuGetPackageRoot)\runtime.$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture).microsoft.netcore.runtime.mono.llvm.tools\$(MonoLLVMSDKVersion)\tools\$(MonoLLVMHostOS)-$(MonoLLVMTargetArchitecture)\**" />
     </ItemGroup>
+    <ItemGroup>
+      <AOTLLVMFiles Include="$(NuGetPackageRoot)\runtime.$(MonoLLVMHostOS)-$(HostArch).microsoft.netcore.runtime.mono.llvm.sdk\$(MonoLLVMSDKVersion)\tools\$(MonoLLVMHostOS)-$(HostArch)\**" />
+      <AOTLLVMFiles Include="$(NuGetPackageRoot)\runtime.$(MonoLLVMHostOS)-$(HostArch).microsoft.netcore.runtime.mono.llvm.tools\$(MonoLLVMSDKVersion)\tools\$(MonoLLVMHostOS)-$(HostArch)\**" />
+    </ItemGroup>
 
     <Copy SourceFiles="@(LLVMFiles)" DestinationFolder="$(MonoLLVMDir)\%(RecursiveDir)">
       <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
     </Copy>
+
+    <Copy SourceFiles="@(AOTLLVMFiles)" DestinationFolder="$(MonoAOTLLVMDir)\%(RecursiveDir)">
+      <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
+    </Copy>
   </Target>
 
 </Project>