[release/7.0] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,14 +3,14 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22379.10">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22558.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1e73f4ab4c172aa55614f24b2d5c319e1efb8813</Sha>
+      <Sha>3f3c360819c5c092d0e4505a67dfe59a33fba557</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.ApiCompat" Version="7.0.0-beta.22379.10">
+    <Dependency Name="Microsoft.DotNet.ApiCompat" Version="7.0.0-beta.22558.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1e73f4ab4c172aa55614f24b2d5c319e1efb8813</Sha>
+      <Sha>3f3c360819c5c092d0e4505a67dfe59a33fba557</Sha>
     </Dependency>
     <Dependency Name="Microsoft.NET.Sdk.IL" Version="7.0.0-rc.1.22407.4">
       <Uri>https://github.com/dotnet/runtime</Uri>

eng/Versions.props

@@ -7,7 +7,6 @@
     <PatchVersion>100</PatchVersion>
     <!-- Use an unchanging prerelease label so it doesn't need to be updated. -->
     <PreReleaseVersionLabel>1</PreReleaseVersionLabel>
-    <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
   </PropertyGroup>
   <PropertyGroup>
     <!-- ilasm -->
@@ -18,11 +17,10 @@
     <SystemReflectionMetadataVersion>5.0.0</SystemReflectionMetadataVersion>
     <MicrosoftBuildFrameworkVersion>17.0.0-preview-21267-01</MicrosoftBuildFrameworkVersion>
     <MicrosoftBuildUtilitiesCoreVersion>17.0.0-preview-21267-01</MicrosoftBuildUtilitiesCoreVersion>
-    <MicrosoftDotNetApiCompatVersion>7.0.0-beta.22379.10</MicrosoftDotNetApiCompatVersion>
+    <MicrosoftDotNetApiCompatVersion>7.0.0-beta.22558.4</MicrosoftDotNetApiCompatVersion>
     <MicrosoftDotNetCodeAnalysisVersion>6.0.0-beta.21271.1</MicrosoftDotNetCodeAnalysisVersion>
     <MicrosoftCodeAnalysisCSharpCodeStyleVersion>3.10.0-2.final</MicrosoftCodeAnalysisCSharpCodeStyleVersion>
     <MicrosoftCodeAnalysisVersion>4.4.0-2.22379.3</MicrosoftCodeAnalysisVersion>
-    <MicrosoftNetCompilersToolsetVersion>$(MicrosoftCodeAnalysisVersion)</MicrosoftNetCompilersToolsetVersion>
     <MicrosoftCodeAnalysisCSharpAnalyzerTestingXunitVersion>1.0.1-beta1.*</MicrosoftCodeAnalysisCSharpAnalyzerTestingXunitVersion>
     <MicrosoftCodeAnalysisBannedApiAnalyzersVersion>3.3.2</MicrosoftCodeAnalysisBannedApiAnalyzersVersion>
     <MicrosoftILVerificationVersion>7.0.0-preview.7.22375.6</MicrosoftILVerificationVersion>

eng/common/build.ps1

@@ -26,6 +26,7 @@ Param(
   [string] $runtimeSourceFeed = '',
   [string] $runtimeSourceFeedKey = '',
   [switch] $excludePrereleaseVS,
+  [switch] $nativeToolsOnMachine,
   [switch] $help,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
@@ -66,6 +67,7 @@ function Print-Usage() {
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host "  -nativeToolsOnMachine   Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
   Write-Host "  -excludePrereleaseVS    Set to exclude build engines in prerelease versions of Visual Studio"
   Write-Host ""
 
@@ -146,6 +148,9 @@ try {
     $nodeReuse = $false
   }
 
+  if ($nativeToolsOnMachine) {
+    $env:NativeToolsOnMachine = $true
+  }
   if ($restore) {
     InitializeNativeTools
   }

eng/common/cross/arm/sources.list.focal

@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

eng/common/cross/arm/sources.list.jammy

@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.focal

@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.jammy

@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

eng/common/cross/build-rootfs.sh

@@ -8,11 +8,13 @@ usage()
     echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
     echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
     echo "                              for FreeBSD can be: freebsd12, freebsd13"
-    echo "                              for illumos can be: illumos."
+    echo "                              for illumos can be: illumos"
+    echo "                                for Haiku can be: haiku."
     echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
     echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
     echo "--skipunmount - optional, will skip the unmount of rootfs folder."
     echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
+    echo "--jobs N - optional, restrict to N jobs."
     exit 1
 }
 
@@ -74,10 +76,21 @@ __FreeBSDPackages+=" openssl"
 __FreeBSDPackages+=" krb5"
 __FreeBSDPackages+=" terminfo-db"
 
-__IllumosPackages="icu-64.2nb2"
-__IllumosPackages+=" mit-krb5-1.16.2nb4"
-__IllumosPackages+=" openssl-1.1.1e"
-__IllumosPackages+=" zlib-1.2.11"
+__IllumosPackages="icu"
+__IllumosPackages+=" mit-krb5"
+__IllumosPackages+=" openssl"
+__IllumosPackages+=" zlib"
+
+__HaikuPackages="gmp"
+__HaikuPackages+=" gmp_devel"
+__HaikuPackages+=" krb5"
+__HaikuPackages+=" krb5_devel"
+__HaikuPackages+=" libiconv"
+__HaikuPackages+=" libiconv_devel"
+__HaikuPackages+=" llvm12_libunwind"
+__HaikuPackages+=" llvm12_libunwind_devel"
+__HaikuPackages+=" mpfr"
+__HaikuPackages+=" mpfr_devel"
 
 # ML.NET dependencies
 __UbuntuPackages+=" libomp5"
@@ -173,32 +186,27 @@ while :; do
             __UbuntuArch=i386
             __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
             ;;
-        lldb3.6)
-            __LLDB_Package="lldb-3.6-dev"
-            ;;
-        lldb3.8)
-            __LLDB_Package="lldb-3.8-dev"
-            ;;
-        lldb3.9)
-            __LLDB_Package="liblldb-3.9-dev"
-            ;;
-        lldb4.0)
-            __LLDB_Package="liblldb-4.0-dev"
-            ;;
-        lldb5.0)
-            __LLDB_Package="liblldb-5.0-dev"
-            ;;
-        lldb6.0)
-            __LLDB_Package="liblldb-6.0-dev"
+        lldb*)
+            version="${lowerI/lldb/}"
+            parts=(${version//./ })
+
+            # for versions > 6.0, lldb has dropped the minor version
+            if [[ "${parts[0]}" -gt 6 ]]; then
+                version="${parts[0]}"
+            fi
+
+            __LLDB_Package="liblldb-${version}-dev"
             ;;
         no-lldb)
             unset __LLDB_Package
             ;;
         llvm*)
-            version="$(echo "$lowerI" | tr -d '[:alpha:]-=')"
+            version="${lowerI/llvm/}"
             parts=(${version//./ })
             __LLVM_MajorVersion="${parts[0]}"
             __LLVM_MinorVersion="${parts[1]}"
+
+            # for versions > 6.0, llvm has dropped the minor version
             if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then
                 __LLVM_MinorVersion=0;
             fi
@@ -218,6 +226,16 @@ while :; do
                 __CodeName=bionic
             fi
             ;;
+        focal) # Ubuntu 20.04
+            if [[ "$__CodeName" != "jessie" ]]; then
+                __CodeName=focal
+            fi
+            ;;
+        jammy) # Ubuntu 22.04
+            if [[ "$__CodeName" != "jessie" ]]; then
+                __CodeName=jammy
+            fi
+            ;;
         jessie) # Debian 8
             __CodeName=jessie
             __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -263,6 +281,11 @@ while :; do
             __CodeName=illumos
             __SkipUnmount=1
             ;;
+        haiku)
+            __CodeName=haiku
+            __BuildArch=x64
+            __SkipUnmount=1
+            ;;
         --skipunmount)
             __SkipUnmount=1
             ;;
@@ -273,6 +296,10 @@ while :; do
         --use-mirror)
             __UseMirror=1
             ;;
+        --use-jobs)
+            shift
+            MAXJOBS=$1
+            ;;
         *)
             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
             ;;
@@ -326,7 +353,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
     rm -r "$__ApkToolsDir"
 elif [[ "$__CodeName" == "freebsd" ]]; then
     mkdir -p "$__RootfsDir"/usr/local/etc
-    JOBS="$(getconf _NPROCESSORS_ONLN)"
+    JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
     wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
     echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
     echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
@@ -344,7 +371,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
 elif [[ "$__CodeName" == "illumos" ]]; then
     mkdir "$__RootfsDir/tmp"
     pushd "$__RootfsDir/tmp"
-    JOBS="$(getconf _NPROCESSORS_ONLN)"
+    JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
     echo "Downloading sysroot."
     wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
     echo "Building binutils. Please wait.."
@@ -368,14 +395,18 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=http://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl/packages/SmartOS/2020Q1/${__illumosArch}/All"
+    BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
+    echo "Downloading manifest"
+    wget "$BaseUrl"
     echo "Downloading dependencies."
     read -ra array <<<"$__IllumosPackages"
     for package in "${array[@]}"; do
-       echo "Installing $package..."
+        echo "Installing '$package'"
+        package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')"
+        echo "Resolved name '$package'"
         wget "$BaseUrl"/"$package".tgz
         ar -x "$package".tgz
-        tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null
+        tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
     done
     echo "Cleaning up temporary files."
     popd
@@ -386,6 +417,70 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
     wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
     wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+elif [[ "$__CodeName" == "haiku" ]]; then
+    JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
+
+    echo "Building Haiku sysroot for x86_64"
+    mkdir -p "$__RootfsDir/tmp"
+    cd "$__RootfsDir/tmp"
+    git clone -b hrev56235  https://review.haiku-os.org/haiku
+    git clone -b btrev43195 https://review.haiku-os.org/buildtools
+    cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d
+
+    # Fetch some unmerged patches
+    cd "$__RootfsDir/tmp/haiku"
+    ## Add development build profile (slimmer than nightly)
+    git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD
+
+    # Build jam
+    cd "$__RootfsDir/tmp/buildtools/jam"
+    make
+
+    # Configure cross tools
+    echo "Building cross-compiler"
+    mkdir -p "$__RootfsDir/generated"
+    cd "$__RootfsDir/generated"
+    "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64
+
+    # Build Haiku packages
+    echo "Building Haiku"
+    echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig
+    "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q '<build>package' '<repository>Haiku'
+
+    BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
+
+    # Download additional packages
+    echo "Downloading additional required packages"
+    read -ra array <<<"$__HaikuPackages"
+    for package in "${array[@]}"; do
+        echo "Downloading $package..."
+        # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
+        # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
+        hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \
+            --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
+        wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl"
+    done
+
+    # Setup the sysroot
+    echo "Setting up sysroot and extracting needed packages"
+    mkdir -p "$__RootfsDir/boot/system"
+    for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do
+        "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
+    done
+    for file in "$__RootfsDir/generated/download/"*.hpkg; do
+        "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
+    done
+
+    # Cleaning up temporary files
+    echo "Cleaning up temporary files"
+    rm -rf "$__RootfsDir/tmp"
+    for name in "$__RootfsDir/generated/"*; do
+        if [[ "$name" =~ "cross-tools-" ]]; then
+            : # Keep the cross-compiler
+        else
+            rm -rf "$name"
+        fi
+    done
 elif [[ -n "$__CodeName" ]]; then
     qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
     cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list"