diff --git a/build.cmd b/build.cmd
index 9aef81fd1f..567e14ac08 100644
--- a/build.cmd
+++ b/build.cmd
@@ -1,3 +1,3 @@
@echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -warnAsError 0 %*"
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\build.ps1""" -restore -build -warnAsError 0 %*"
exit /b %ErrorLevel%
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index e348d0998a..1d6c1d91a6 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -1,43 +1,43 @@
-
+
https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader
- 9bdfdb0af37d2e93bdecf238a8a51c0a965444d6
+ 8a3aaf73bc04edffd49429f44e35a70d2ba9e9c9
-
+
https://github.com/dotnet/arcade
- a81e6e87cf21837abfde2da6eb9b057bea9f49fc
+ e9fd640e1ec1890489ea66ad0f59e733448056da
-
+
https://github.com/dotnet/arcade
- a81e6e87cf21837abfde2da6eb9b057bea9f49fc
+ e9fd640e1ec1890489ea66ad0f59e733448056da
-
+
https://github.com/dotnet/arcade
- a81e6e87cf21837abfde2da6eb9b057bea9f49fc
+ e9fd640e1ec1890489ea66ad0f59e733448056da
-
+
https://github.com/dotnet/arcade
- a81e6e87cf21837abfde2da6eb9b057bea9f49fc
+ e9fd640e1ec1890489ea66ad0f59e733448056da
-
+
https://github.com/dotnet/arcade
- a81e6e87cf21837abfde2da6eb9b057bea9f49fc
+ e9fd640e1ec1890489ea66ad0f59e733448056da
-
+
https://github.com/dotnet/arcade-services
- 869869342f1ec338de96adcea6e003b61f195256
+ cd705029f2675970b42f9273ae359d0926c5e815
-
+
https://github.com/dotnet/arcade-services
- 592654f4a6855d7738a7c7c780355ac54457fdae
+ cd705029f2675970b42f9273ae359d0926c5e815
-
+
https://github.com/dotnet/xharness
- 3b64ab7ab565cfd19fe7102e3d76271f16f0fc6d
+ 89cb4b1d368e0f15b4df8e02a176dd1f1c33958b
https://github.com/dotnet/roslyn
@@ -59,9 +59,9 @@
https://github.com/dotnet/symreader-converter
c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0
-
+
https://github.com/dotnet/xliff-tasks
- 975065e08307a459dc2649b1c852f5c4cafd2f91
+ 2739f40a72dec8100f99be106ae6b7d640e6fe69
diff --git a/eng/Versions.props b/eng/Versions.props
index 96685cc388..b6917f28ce 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -12,13 +12,11 @@
1.5.6
preview
1.0.0.0
-
10.0.3
4.4.0
4.3.0
4.7.1
-
3.10.1
2.2.3
@@ -39,8 +37,7 @@
4.3.0
4.5.0
1.2.0
-
-
+
1.0.0-beta-62824-02
1.9.0
1.2.1
@@ -48,7 +45,6 @@
1.0.0-beta.19225.5
1.1.0-beta-20206-02
3.0.0-preview4-04926-01
-
0.12.0
1.0.1-beta1.20374.2
@@ -60,7 +56,6 @@
4.6.1
1.2.7
1.0.112.2
-
false
true
@@ -118,8 +113,8 @@
2.4.0
2.0.3
2.4.0
- 5.0.0-beta.20461.7
- 5.0.0-beta.20461.7
+ 6.0.0-beta.21253.2
+ 6.0.0-beta.21253.2
1.22.0
1.1.2
2.0.0
@@ -129,13 +124,13 @@
1.1.0-beta2-19575-01
1.1.0-beta2-19575-01
1.7.0
- 1.1.0-beta.20258.6
+ 1.1.0-beta.20074.1
1.1.0-beta-20464-02
1.1.0-beta-20464-02
- 5.0.0-beta.20461.7
- 1.0.0-beta.20420.1
- 1.1.0-beta.20461.2
- 1.0.0-prerelease.20457.1
- 1.1.145102
+ 6.0.0-beta.21253.2
+ 1.0.0-beta.21254.1
+ 1.1.0-beta.20074.1
+ 1.0.0-prerelease.21254.2
+ 1.1.225302
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index bb3617133f..a0b5fc37f4 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -99,8 +99,9 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Passw
function EnablePrivatePackageSources($DisabledPackageSources) {
$maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroPrivateSources) {
- Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled"
- $DisabledPackageSource.SetAttribute("value", "false")
+ Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+ # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
+ $DisabledPackageSources.RemoveChild($DisabledPackageSource)
}
}
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
index ef33382954..2734601c13 100755
--- a/eng/common/SetupNugetSources.sh
+++ b/eng/common/SetupNugetSources.sh
@@ -158,8 +158,8 @@ if [ "$?" == "0" ]; then
for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
if [[ $DisabledSourceName == darc-int* ]]
then
- OldDisableValue="add key=\"$DisabledSourceName\" value=\"true\""
- NewDisableValue="add key=\"$DisabledSourceName\" value=\"false\""
+ OldDisableValue=""
+ NewDisableValue=""
sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
fi
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 1fd7f686fa..94a91c0817 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -7,7 +7,6 @@ Param(
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
- [bool] $useDefaultDotnetInstall = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@@ -66,7 +65,6 @@ function Print-Usage() {
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
- Write-Host " -useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
diff --git a/eng/common/build.sh b/eng/common/build.sh
index 19849adbee..55b298f16c 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -36,8 +36,6 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
- echo " --useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder"
-
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@@ -80,11 +78,10 @@ prepare_machine=false
verbosity='minimal'
runtime_source_feed=''
runtime_source_feed_key=''
-use_default_dotnet_install=false
properties=''
while [[ $# > 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
usage
@@ -159,14 +156,10 @@ while [[ $# > 0 ]]; do
runtime_source_feed=$2
shift
;;
- -runtimesourcefeedkey)
+ -runtimesourcefeedkey)
runtime_source_feed_key=$2
shift
;;
- -usedefaultdotnetinstall)
- use_default_dotnet_install=$2
- shift
- ;;
*)
properties="$properties $1"
;;
diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh
index a48a6f51c4..16d1301f21 100755
--- a/eng/common/cross/arm64/tizen-fetch.sh
+++ b/eng/common/cross/arm64/tizen-fetch.sh
@@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
-fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel
+fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
new file mode 100644
index 0000000000..2d26156193
--- /dev/null
+++ b/eng/common/cross/armel/armel.jessie.patch
@@ -0,0 +1,43 @@
+diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
+--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
++++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
+@@ -69,10 +69,10 @@
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+- return __sync_val_compare_and_swap_2(addr, old, _new);
++ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
+ #endif
+ case 4:
+- return __sync_val_compare_and_swap_4(addr, old, _new);
++ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_val_compare_and_swap_8(addr, old, _new);
+@@ -109,7 +109,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_and_and_fetch_4(addr, val);
++ __sync_and_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -148,7 +148,7 @@
+ return;
+ #endif
+ case 4:
+- __sync_or_and_fetch_4(addr, val);
++ __sync_or_and_fetch_4((uint32_t*) addr, val);
+ return;
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+@@ -187,7 +187,7 @@
+ return __sync_add_and_fetch_2(addr, val);
+ #endif
+ case 4:
+- return __sync_add_and_fetch_4(addr, val);
++ return __sync_add_and_fetch_4((uint32_t*) addr, val);
+ #if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh
index 2776cbba4e..64f0187e5a 100755
--- a/eng/common/cross/armel/tizen-fetch.sh
+++ b/eng/common/cross/armel/tizen-fetch.sh
@@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
-fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel
+fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index e7f12edb56..c29c8267e7 100755
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
for i in "$@"
do
- lowerI="$(echo $i | awk '{print tolower($0)}')"
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
@@ -106,6 +106,7 @@ __AndroidPackages+=" libandroid-glob"
__AndroidPackages+=" liblzma"
__AndroidPackages+=" krb5"
__AndroidPackages+=" openssl"
+__AndroidPackages+=" openldap"
for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index ffdff38542..81e641a57b 100755
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -6,7 +6,7 @@ usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]"
echo "BuildArch can be: arm(default), armel, arm64, x86"
- echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
+ echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11 or freebsd12."
echo " for illumos can be: illumos."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
@@ -55,11 +55,13 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev"
__UbuntuPackages+=" zlib1g-dev"
+__UbuntuPackages+=" libldap2-dev"
__AlpinePackages+=" curl-dev"
__AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
+__AlpinePackages+=" openldap-dev"
__FreeBSDBase="12.1-RELEASE"
__FreeBSDPkg="1.12.0"
@@ -68,11 +70,13 @@ __FreeBSDPackages+=" icu"
__FreeBSDPackages+=" libinotify"
__FreeBSDPackages+=" lttng-ust"
__FreeBSDPackages+=" krb5"
+__FreeBSDPackages+=" libslapi-2.4"
__IllumosPackages="icu-64.2nb2"
__IllumosPackages+=" mit-krb5-1.16.2nb4"
__IllumosPackages+=" openssl-1.1.1e"
__IllumosPackages+=" zlib-1.2.11"
+__IllumosPackages+=" openldap-client-2.4.49"
__UseMirror=0
@@ -82,7 +86,7 @@ while :; do
break
fi
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
@@ -106,6 +110,13 @@ while :; do
__UbuntuRepo="http://ftp.debian.org/debian/"
__CodeName=jessie
;;
+ s390x)
+ __BuildArch=s390x
+ __UbuntuArch=s390x
+ __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+ __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+ unset __LLDB_Package
+ ;;
x86)
__BuildArch=x86
__UbuntuArch=i386
@@ -176,9 +187,20 @@ while :; do
__UbuntuRepo=
__Tizen=tizen
;;
- alpine)
+ alpine|alpine3.9)
+ __CodeName=alpine
+ __UbuntuRepo=
+ __AlpineVersion=3.9
+ ;;
+ alpine3.13)
__CodeName=alpine
__UbuntuRepo=
+ __AlpineVersion=3.13
+ # Alpine 3.13 has all the packages we need in the 3.13 repository
+ __AlpinePackages+=$__AlpinePackagesEdgeCommunity
+ __AlpinePackagesEdgeCommunity=
+ __AlpinePackages+=$__AlpinePackagesEdgeMain
+ __AlpinePackagesEdgeMain=
;;
freebsd11)
__FreeBSDBase="11.3-RELEASE"
@@ -236,7 +258,6 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1
- __AlpineVersion=3.9
__ApkToolsDir=$(mktemp -d)
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
@@ -249,15 +270,19 @@ if [[ "$__CodeName" == "alpine" ]]; then
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages
- $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
- -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
- -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
- add $__AlpinePackagesEdgeMain
+ if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
+ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+ -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
+ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+ add $__AlpinePackagesEdgeMain
+ fi
- $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
- -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
- -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
- add $__AlpinePackagesEdgeCommunity
+ if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
+ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+ -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
+ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+ add $__AlpinePackagesEdgeCommunity
+ fi
rm -r $__ApkToolsDir
elif [[ "$__CodeName" == "freebsd" ]]; then
@@ -329,6 +354,7 @@ elif [[ -n $__CodeName ]]; then
chroot $__RootfsDir apt-get -f -y install
chroot $__RootfsDir apt-get -y install $__UbuntuPackages
chroot $__RootfsDir symlinks -cr /usr
+ chroot $__RootfsDir apt-get clean
if [ $__SkipUnmount == 0 ]; then
umount $__RootfsDir/* || true
@@ -340,6 +366,12 @@ elif [[ -n $__CodeName ]]; then
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
popd
fi
+
+ if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
+ pushd $__RootfsDir
+ patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
+ popd
+ fi
elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
else
diff --git a/eng/common/cross/s390x/sources.list.bionic b/eng/common/cross/s390x/sources.list.bionic
new file mode 100644
index 0000000000..2109557409
--- /dev/null
+++ b/eng/common/cross/s390x/sources.list.bionic
@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index 137736c0a2..fc11001aa7 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -36,6 +36,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
if("$ENV{__DistroRid}" MATCHES "tizen.*")
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+ set(CMAKE_SYSTEM_PROCESSOR s390x)
+ set(TOOLCHAIN "s390x-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(CMAKE_SYSTEM_PROCESSOR i686)
set(TOOLCHAIN "i686-linux-gnu")
@@ -46,7 +49,7 @@ elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos")
else()
- message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
+ message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
endif()
if(DEFINED ENV{TOOLCHAIN})
@@ -139,6 +142,10 @@ function(add_toolchain_linker_flag Flag)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
endfunction()
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
+ add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}")
+endif()
if(TARGET_ARCH_NAME STREQUAL "armel")
if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
@@ -167,7 +174,7 @@ endif()
# Specify compile options
-if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
+if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index d981d7bbf3..39abdbecdc 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
verbosity='minimal'
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)
darcVersion=$2
diff --git a/eng/common/dotnet-install-scripts/dotnet-install.ps1 b/eng/common/dotnet-install-scripts/dotnet-install.ps1
deleted file mode 100644
index f63b533f25..0000000000
--- a/eng/common/dotnet-install-scripts/dotnet-install.ps1
+++ /dev/null
@@ -1,774 +0,0 @@
-#
-# Copyright (c) .NET Foundation and contributors. All rights reserved.
-# Licensed under the MIT license. See LICENSE file in the project root for full license information.
-#
-
-# Copied from https://dot.net/v1/dotnet-install.ps1 on 8/26/2020
-
-<#
-.SYNOPSIS
- Installs dotnet cli
-.DESCRIPTION
- Installs dotnet cli. If dotnet installation already exists in the given directory
- it will update it only if the requested version differs from the one already installed.
-.PARAMETER Channel
- Default: LTS
- Download from the Channel specified. Possible values:
- - Current - most current release
- - LTS - most current supported release
- - 2-part version in a format A.B - represents a specific release
- examples: 2.0, 1.0
- - Branch name
- examples: release/2.0.0, Master
- Note: The version parameter overrides the channel parameter.
-.PARAMETER Version
- Default: latest
- Represents a build version on specific channel. Possible values:
- - latest - most latest build on specific channel
- - coherent - most latest coherent build on specific channel
- coherent applies only to SDK downloads
- - 3-part version in a format A.B.C - represents specific version of build
- examples: 2.0.0-preview2-006120, 1.1.0
-.PARAMETER InstallDir
- Default: %LocalAppData%\Microsoft\dotnet
- Path to where to install dotnet. Note that binaries will be placed directly in a given directory.
-.PARAMETER Architecture
- Default: - this value represents currently running OS architecture
- Architecture of dotnet binaries to be installed.
- Possible values are: , amd64, x64, x86, arm64, arm
-.PARAMETER SharedRuntime
- This parameter is obsolete and may be removed in a future version of this script.
- The recommended alternative is '-Runtime dotnet'.
- Installs just the shared runtime bits, not the entire SDK.
-.PARAMETER Runtime
- Installs just a shared runtime, not the entire SDK.
- Possible values:
- - dotnet - the Microsoft.NETCore.App shared runtime
- - aspnetcore - the Microsoft.AspNetCore.App shared runtime
- - windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime
-.PARAMETER DryRun
- If set it will not perform installation but instead display what command line to use to consistently install
- currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link
- with specific version so that this command can be used deterministicly in a build script.
- It also displays binaries location if you prefer to install or download it yourself.
-.PARAMETER NoPath
- By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder.
- If set it will display binaries location but not set any environment variable.
-.PARAMETER Verbose
- Displays diagnostics information.
-.PARAMETER AzureFeed
- Default: https://dotnetcli.azureedge.net/dotnet
- This parameter typically is not changed by the user.
- It allows changing the URL for the Azure feed used by this installer.
-.PARAMETER UncachedFeed
- This parameter typically is not changed by the user.
- It allows changing the URL for the Uncached feed used by this installer.
-.PARAMETER FeedCredential
- Used as a query string to append to the Azure feed.
- It allows changing the URL to use non-public blob storage accounts.
-.PARAMETER ProxyAddress
- If set, the installer will use the proxy when making web requests
-.PARAMETER ProxyUseDefaultCredentials
- Default: false
- Use default credentials, when using proxy address.
-.PARAMETER ProxyBypassList
- If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy
-.PARAMETER SkipNonVersionedFiles
- Default: false
- Skips installing non-versioned files if they already exist, such as dotnet.exe.
-.PARAMETER NoCdn
- Disable downloading from the Azure CDN, and use the uncached feed directly.
-.PARAMETER JSonFile
- Determines the SDK version from a user specified global.json file
- Note: global.json must have a value for 'SDK:Version'
-#>
-[cmdletbinding()]
-param(
- [string]$Channel="LTS",
- [string]$Version="Latest",
- [string]$JSonFile,
- [string]$InstallDir="",
- [string]$Architecture="",
- [ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)]
- [string]$Runtime,
- [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")]
- [switch]$SharedRuntime,
- [switch]$DryRun,
- [switch]$NoPath,
- [string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet",
- [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet",
- [string]$FeedCredential,
- [string]$ProxyAddress,
- [switch]$ProxyUseDefaultCredentials,
- [string[]]$ProxyBypassList=@(),
- [switch]$SkipNonVersionedFiles,
- [switch]$NoCdn
-)
-
-Set-StrictMode -Version Latest
-$ErrorActionPreference="Stop"
-$ProgressPreference="SilentlyContinue"
-
-if ($NoCdn) {
- $AzureFeed = $UncachedFeed
-}
-
-$BinFolderRelativePath=""
-
-if ($SharedRuntime -and (-not $Runtime)) {
- $Runtime = "dotnet"
-}
-
-# example path with regex: shared/1.0.0-beta-12345/somepath
-$VersionRegEx="/\d+\.\d+[^/]+/"
-$OverrideNonVersionedFiles = !$SkipNonVersionedFiles
-
-function Say($str) {
- try
- {
- Write-Host "dotnet-install: $str"
- }
- catch
- {
- # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output
- Write-Output "dotnet-install: $str"
- }
-}
-
-function Say-Verbose($str) {
- try
- {
- Write-Verbose "dotnet-install: $str"
- }
- catch
- {
- # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output
- Write-Output "dotnet-install: $str"
- }
-}
-
-function Say-Invocation($Invocation) {
- $command = $Invocation.MyCommand;
- $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ")
- Say-Verbose "$command $args"
-}
-
-function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) {
- $Attempts = 0
-
- while ($true) {
- try {
- return $ScriptBlock.Invoke()
- }
- catch {
- $Attempts++
- if ($Attempts -lt $MaxAttempts) {
- Start-Sleep $SecondsBetweenAttempts
- }
- else {
- throw
- }
- }
- }
-}
-
-function Get-Machine-Architecture() {
- Say-Invocation $MyInvocation
-
- # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems.
- # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432.
- # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE.
- # Possible values: amd64, x64, x86, arm64, arm
-
- if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null )
- {
- return $ENV:PROCESSOR_ARCHITEW6432
- }
-
- return $ENV:PROCESSOR_ARCHITECTURE
-}
-
-function Get-CLIArchitecture-From-Architecture([string]$Architecture) {
- Say-Invocation $MyInvocation
-
- switch ($Architecture.ToLower()) {
- { $_ -eq "" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) }
- { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" }
- { $_ -eq "x86" } { return "x86" }
- { $_ -eq "arm" } { return "arm" }
- { $_ -eq "arm64" } { return "arm64" }
- default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" }
- }
-}
-
-# The version text returned from the feeds is a 1-line or 2-line string:
-# For the SDK and the dotnet runtime (2 lines):
-# Line 1: # commit_hash
-# Line 2: # 4-part version
-# For the aspnetcore runtime (1 line):
-# Line 1: # 4-part version
-function Get-Version-Info-From-Version-Text([string]$VersionText) {
- Say-Invocation $MyInvocation
-
- $Data = -split $VersionText
-
- $VersionInfo = @{
- CommitHash = $(if ($Data.Count -gt 1) { $Data[0] })
- Version = $Data[-1] # last line is always the version number.
- }
- return $VersionInfo
-}
-
-function Load-Assembly([string] $Assembly) {
- try {
- Add-Type -Assembly $Assembly | Out-Null
- }
- catch {
- # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd.
- # Loading the base class assemblies is not unnecessary as the types will automatically get resolved.
- }
-}
-
-function GetHTTPResponse([Uri] $Uri)
-{
- Invoke-With-Retry(
- {
-
- $HttpClient = $null
-
- try {
- # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet.
- Load-Assembly -Assembly System.Net.Http
-
- if(-not $ProxyAddress) {
- try {
- # Despite no proxy being explicitly specified, we may still be behind a default proxy
- $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy;
- if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) {
- $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString
- $ProxyUseDefaultCredentials = $true
- }
- } catch {
- # Eat the exception and move forward as the above code is an attempt
- # at resolving the DefaultProxy that may not have been a problem.
- $ProxyAddress = $null
- Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...")
- }
- }
-
- if($ProxyAddress) {
- $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler
- $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{
- Address=$ProxyAddress;
- UseDefaultCredentials=$ProxyUseDefaultCredentials;
- BypassList = $ProxyBypassList;
- }
- $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler
- }
- else {
-
- $HttpClient = New-Object System.Net.Http.HttpClient
- }
- # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out
- # 20 minutes allows it to work over much slower connections.
- $HttpClient.Timeout = New-TimeSpan -Minutes 20
- $Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result
- if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) {
- # The feed credential is potentially sensitive info. Do not log FeedCredential to console output.
- $ErrorMsg = "Failed to download $Uri."
- if ($Response -ne $null) {
- $ErrorMsg += " $Response"
- }
-
- throw $ErrorMsg
- }
-
- return $Response
- }
- finally {
- if ($HttpClient -ne $null) {
- $HttpClient.Dispose()
- }
- }
- })
-}
-
-function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) {
- Say-Invocation $MyInvocation
-
- $VersionFileUrl = $null
- if ($Runtime -eq "dotnet") {
- $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
- }
- elseif ($Runtime -eq "aspnetcore") {
- $VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version"
- }
- # Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime
- elseif ($Runtime -eq "windowsdesktop") {
- $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
- }
- elseif (-not $Runtime) {
- if ($Coherent) {
- $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version"
- }
- else {
- $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version"
- }
- }
- else {
- throw "Invalid value for `$Runtime"
- }
- try {
- $Response = GetHTTPResponse -Uri $VersionFileUrl
- }
- catch {
- throw "Could not resolve version information."
- }
- $StringContent = $Response.Content.ReadAsStringAsync().Result
-
- switch ($Response.Content.Headers.ContentType) {
- { ($_ -eq "application/octet-stream") } { $VersionText = $StringContent }
- { ($_ -eq "text/plain") } { $VersionText = $StringContent }
- { ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent }
- default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." }
- }
-
- $VersionInfo = Get-Version-Info-From-Version-Text $VersionText
-
- return $VersionInfo
-}
-
-function Parse-Jsonfile-For-Version([string]$JSonFile) {
- Say-Invocation $MyInvocation
-
- If (-Not (Test-Path $JSonFile)) {
- throw "Unable to find '$JSonFile'"
- }
- try {
- $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue
- }
- catch {
- throw "Json file unreadable: '$JSonFile'"
- }
- if ($JSonContent) {
- try {
- $JSonContent.PSObject.Properties | ForEach-Object {
- $PropertyName = $_.Name
- if ($PropertyName -eq "version") {
- $Version = $_.Value
- Say-Verbose "Version = $Version"
- }
- }
- }
- catch {
- throw "Unable to parse the SDK node in '$JSonFile'"
- }
- }
- else {
- throw "Unable to find the SDK node in '$JSonFile'"
- }
- If ($Version -eq $null) {
- throw "Unable to find the SDK:version node in '$JSonFile'"
- }
- return $Version
-}
-
-function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) {
- Say-Invocation $MyInvocation
-
- if (-not $JSonFile) {
- switch ($Version.ToLower()) {
- { $_ -eq "latest" } {
- $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False
- return $LatestVersionInfo.Version
- }
- { $_ -eq "coherent" } {
- $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
- return $LatestVersionInfo.Version
- }
- default { return $Version }
- }
- }
- else {
- return Parse-Jsonfile-For-Version $JSonFile
- }
-}
-
-function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
- Say-Invocation $MyInvocation
-
- # If anything fails in this lookup it will default to $SpecificVersion
- $SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion
-
- if ($Runtime -eq "dotnet") {
- $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
- }
- elseif ($Runtime -eq "aspnetcore") {
- $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
- }
- elseif ($Runtime -eq "windowsdesktop") {
- $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
- }
- elseif (-not $Runtime) {
- $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip"
- }
- else {
- throw "Invalid value for `$Runtime"
- }
-
- Say-Verbose "Constructed primary named payload URL: $PayloadURL"
-
- return $PayloadURL, $SpecificProductVersion
-}
-
-function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
- Say-Invocation $MyInvocation
-
- if (-not $Runtime) {
- $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip"
- }
- elseif ($Runtime -eq "dotnet") {
- $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip"
- }
- else {
- return $null
- }
-
- Say-Verbose "Constructed legacy named payload URL: $PayloadURL"
-
- return $PayloadURL
-}
-
-function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) {
- Say-Invocation $MyInvocation
-
- if ($Runtime -eq "dotnet") {
- $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
- }
- elseif ($Runtime -eq "aspnetcore") {
- $ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt"
- }
- elseif ($Runtime -eq "windowsdesktop") {
- $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
- }
- elseif (-not $Runtime) {
- $ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt"
- }
- else {
- throw "Invalid value specified for `$Runtime"
- }
-
- Say-Verbose "Checking for existence of $ProductVersionTxtURL"
-
- try {
- $productVersionResponse = GetHTTPResponse($productVersionTxtUrl)
-
- if ($productVersionResponse.StatusCode -eq 200) {
- $productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim()
- if ($productVersion -ne $SpecificVersion)
- {
- Say "Using alternate version $productVersion found in $ProductVersionTxtURL"
- }
-
- return $productVersion
- }
- else {
- Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion"
- $productVersion = $SpecificVersion
- }
- } catch {
- Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion"
- $productVersion = $SpecificVersion
- }
-
- return $productVersion
-}
-
-function Get-User-Share-Path() {
- Say-Invocation $MyInvocation
-
- $InstallRoot = $env:DOTNET_INSTALL_DIR
- if (!$InstallRoot) {
- $InstallRoot = "$env:LocalAppData\Microsoft\dotnet"
- }
- return $InstallRoot
-}
-
-function Resolve-Installation-Path([string]$InstallDir) {
- Say-Invocation $MyInvocation
-
- if ($InstallDir -eq "") {
- return Get-User-Share-Path
- }
- return $InstallDir
-}
-
-function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) {
- Say-Invocation $MyInvocation
-
- $DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion
- Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath"
- return Test-Path $DotnetPackagePath -PathType Container
-}
-
-function Get-Absolute-Path([string]$RelativeOrAbsolutePath) {
- # Too much spam
- # Say-Invocation $MyInvocation
-
- return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath)
-}
-
-function Get-Path-Prefix-With-Version($path) {
- $match = [regex]::match($path, $VersionRegEx)
- if ($match.Success) {
- return $entry.FullName.Substring(0, $match.Index + $match.Length)
- }
-
- return $null
-}
-
-function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) {
- Say-Invocation $MyInvocation
-
- $ret = @()
- foreach ($entry in $Zip.Entries) {
- $dir = Get-Path-Prefix-With-Version $entry.FullName
- if ($dir -ne $null) {
- $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir)
- if (-Not (Test-Path $path -PathType Container)) {
- $ret += $dir
- }
- }
- }
-
- $ret = $ret | Sort-Object | Get-Unique
-
- $values = ($ret | foreach { "$_" }) -join ";"
- Say-Verbose "Directories to unpack: $values"
-
- return $ret
-}
-
-# Example zip content and extraction algorithm:
-# Rule: files if extracted are always being extracted to the same relative path locally
-# .\
-# a.exe # file does not exist locally, extract
-# b.dll # file exists locally, override only if $OverrideFiles set
-# aaa\ # same rules as for files
-# ...
-# abc\1.0.0\ # directory contains version and exists locally
-# ... # do not extract content under versioned part
-# abc\asd\ # same rules as for files
-# ...
-# def\ghi\1.0.1\ # directory contains version and does not exist locally
-# ... # extract content
-function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) {
- Say-Invocation $MyInvocation
-
- Load-Assembly -Assembly System.IO.Compression.FileSystem
- Set-Variable -Name Zip
- try {
- $Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath)
-
- $DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath
-
- foreach ($entry in $Zip.Entries) {
- $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName
- if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) {
- $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName)
- $DestinationDir = Split-Path -Parent $DestinationPath
- $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath))
- if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) {
- New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null
- [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles)
- }
- }
- }
- }
- finally {
- if ($Zip -ne $null) {
- $Zip.Dispose()
- }
- }
-}
-
-function DownloadFile($Source, [string]$OutPath) {
- if ($Source -notlike "http*") {
- # Using System.IO.Path.GetFullPath to get the current directory
- # does not work in this context - $pwd gives the current directory
- if (![System.IO.Path]::IsPathRooted($Source)) {
- $Source = $(Join-Path -Path $pwd -ChildPath $Source)
- }
- $Source = Get-Absolute-Path $Source
- Say "Copying file from $Source to $OutPath"
- Copy-Item $Source $OutPath
- return
- }
-
- $Stream = $null
-
- try {
- $Response = GetHTTPResponse -Uri $Source
- $Stream = $Response.Content.ReadAsStreamAsync().Result
- $File = [System.IO.File]::Create($OutPath)
- $Stream.CopyTo($File)
- $File.Close()
- }
- finally {
- if ($Stream -ne $null) {
- $Stream.Dispose()
- }
- }
-}
-
-function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) {
- $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath)
- if (-Not $NoPath) {
- $SuffixedBinPath = "$BinPath;"
- if (-Not $env:path.Contains($SuffixedBinPath)) {
- Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process."
- $env:path = $SuffixedBinPath + $env:path
- } else {
- Say-Verbose "Current process PATH already contains `"$BinPath`""
- }
- }
- else {
- Say "Binaries of dotnet can be found in $BinPath"
- }
-}
-
-$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture
-$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile
-$DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
-$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
-
-$InstallRoot = Resolve-Installation-Path $InstallDir
-Say-Verbose "InstallRoot: $InstallRoot"
-$ScriptName = $MyInvocation.MyCommand.Name
-
-if ($DryRun) {
- Say "Payload URLs:"
- Say "Primary named payload URL: $DownloadLink"
- if ($LegacyDownloadLink) {
- Say "Legacy named payload URL: $LegacyDownloadLink"
- }
- $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`""
- if ($Runtime -eq "dotnet") {
- $RepeatableCommand+=" -Runtime `"dotnet`""
- }
- elseif ($Runtime -eq "aspnetcore") {
- $RepeatableCommand+=" -Runtime `"aspnetcore`""
- }
- foreach ($key in $MyInvocation.BoundParameters.Keys) {
- if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) {
- $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`""
- }
- }
- Say "Repeatable invocation: $RepeatableCommand"
- exit 0
-}
-
-if ($Runtime -eq "dotnet") {
- $assetName = ".NET Core Runtime"
- $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App"
-}
-elseif ($Runtime -eq "aspnetcore") {
- $assetName = "ASP.NET Core Runtime"
- $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App"
-}
-elseif ($Runtime -eq "windowsdesktop") {
- $assetName = ".NET Core Windows Desktop Runtime"
- $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App"
-}
-elseif (-not $Runtime) {
- $assetName = ".NET Core SDK"
- $dotnetPackageRelativePath = "sdk"
-}
-else {
- throw "Invalid value for `$Runtime"
-}
-
-if ($SpecificVersion -ne $EffectiveVersion)
-{
- Say "Performing installation checks for effective version: $EffectiveVersion"
- $SpecificVersion = $EffectiveVersion
-}
-
-# Check if the SDK version is already installed.
-$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
-if ($isAssetInstalled) {
- Say "$assetName version $SpecificVersion is already installed."
- Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
- exit 0
-}
-
-New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null
-
-$installDrive = $((Get-Item $InstallRoot).PSDrive.Name);
-$diskInfo = Get-PSDrive -Name $installDrive
-if ($diskInfo.Free / 1MB -le 100) {
- Say "There is not enough disk space on drive ${installDrive}:"
- exit 0
-}
-
-$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
-Say-Verbose "Zip path: $ZipPath"
-
-$DownloadFailed = $false
-Say "Downloading link: $DownloadLink"
-try {
- DownloadFile -Source $DownloadLink -OutPath $ZipPath
-}
-catch {
- Say "Cannot download: $DownloadLink"
- if ($LegacyDownloadLink) {
- $DownloadLink = $LegacyDownloadLink
- $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
- Say-Verbose "Legacy zip path: $ZipPath"
- Say "Downloading legacy link: $DownloadLink"
- try {
- DownloadFile -Source $DownloadLink -OutPath $ZipPath
- }
- catch {
- Say "Cannot download: $DownloadLink"
- $DownloadFailed = $true
- }
- }
- else {
- $DownloadFailed = $true
- }
-}
-
-if ($DownloadFailed) {
- throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
-}
-
-Say "Extracting zip from $DownloadLink"
-Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot
-
-# Check if the SDK version is installed; if not, fail the installation.
-$isAssetInstalled = $false
-
-# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed.
-if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") {
- $ReleaseVersion = $SpecificVersion.Split("-")[0]
- Say-Verbose "Checking installation: version = $ReleaseVersion"
- $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion
-}
-
-# Check if the SDK version is installed.
-if (!$isAssetInstalled) {
- Say-Verbose "Checking installation: version = $SpecificVersion"
- $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
-}
-
-if (!$isAssetInstalled) {
- throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error."
-}
-
-Remove-Item $ZipPath
-
-Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
-
-Say "Installation finished"
-exit 0
\ No newline at end of file
diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh
deleted file mode 100755
index 92161141f6..0000000000
--- a/eng/common/dotnet-install-scripts/dotnet-install.sh
+++ /dev/null
@@ -1,1133 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) .NET Foundation and contributors. All rights reserved.
-# Licensed under the MIT license. See LICENSE file in the project root for full license information.
-#
-
-# Stop script on NZEC
-set -e
-# Stop script if unbound variable found (use ${var:-} if intentional)
-set -u
-# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success
-# This is causing it to fail
-set -o pipefail
-
-# Use in the the functions: eval $invocation
-invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"'
-
-# standard output may be used as a return value in the functions
-# we need a way to write text on the screen in the functions so that
-# it won't interfere with the return value.
-# Exposing stream 3 as a pipe to standard output of the script itself
-exec 3>&1
-
-# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors.
-# See if stdout is a terminal
-if [ -t 1 ] && command -v tput > /dev/null; then
- # see if it supports colors
- ncolors=$(tput colors)
- if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then
- bold="$(tput bold || echo)"
- normal="$(tput sgr0 || echo)"
- black="$(tput setaf 0 || echo)"
- red="$(tput setaf 1 || echo)"
- green="$(tput setaf 2 || echo)"
- yellow="$(tput setaf 3 || echo)"
- blue="$(tput setaf 4 || echo)"
- magenta="$(tput setaf 5 || echo)"
- cyan="$(tput setaf 6 || echo)"
- white="$(tput setaf 7 || echo)"
- fi
-fi
-
-say_warning() {
- printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}"
-}
-
-say_err() {
- printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2
-}
-
-say() {
- # using stream 3 (defined in the beginning) to not interfere with stdout of functions
- # which may be used as return value
- printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3
-}
-
-say_verbose() {
- if [ "$verbose" = true ]; then
- say "$1"
- fi
-}
-
-# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets,
-# then and only then should the Linux distribution appear in this list.
-# Adding a Linux distribution to this list does not imply distribution-specific support.
-get_legacy_os_name_from_platform() {
- eval $invocation
-
- platform="$1"
- case "$platform" in
- "centos.7")
- echo "centos"
- return 0
- ;;
- "debian.8")
- echo "debian"
- return 0
- ;;
- "debian.9")
- echo "debian.9"
- return 0
- ;;
- "fedora.23")
- echo "fedora.23"
- return 0
- ;;
- "fedora.24")
- echo "fedora.24"
- return 0
- ;;
- "fedora.27")
- echo "fedora.27"
- return 0
- ;;
- "fedora.28")
- echo "fedora.28"
- return 0
- ;;
- "opensuse.13.2")
- echo "opensuse.13.2"
- return 0
- ;;
- "opensuse.42.1")
- echo "opensuse.42.1"
- return 0
- ;;
- "opensuse.42.3")
- echo "opensuse.42.3"
- return 0
- ;;
- "rhel.7"*)
- echo "rhel"
- return 0
- ;;
- "ubuntu.14.04")
- echo "ubuntu"
- return 0
- ;;
- "ubuntu.16.04")
- echo "ubuntu.16.04"
- return 0
- ;;
- "ubuntu.16.10")
- echo "ubuntu.16.10"
- return 0
- ;;
- "ubuntu.18.04")
- echo "ubuntu.18.04"
- return 0
- ;;
- "alpine.3.4.3")
- echo "alpine"
- return 0
- ;;
- esac
- return 1
-}
-
-get_linux_platform_name() {
- eval $invocation
-
- if [ -n "$runtime_id" ]; then
- echo "${runtime_id%-*}"
- return 0
- else
- if [ -e /etc/os-release ]; then
- . /etc/os-release
- echo "$ID${VERSION_ID:+.${VERSION_ID}}"
- return 0
- elif [ -e /etc/redhat-release ]; then
- local redhatRelease=$(&1 || true) | grep -q musl
-}
-
-get_current_os_name() {
- eval $invocation
-
- local uname=$(uname)
- if [ "$uname" = "Darwin" ]; then
- echo "osx"
- return 0
- elif [ "$uname" = "FreeBSD" ]; then
- echo "freebsd"
- return 0
- elif [ "$uname" = "Linux" ]; then
- local linux_platform_name
- linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; }
-
- if [ "$linux_platform_name" = "rhel.6" ]; then
- echo $linux_platform_name
- return 0
- elif is_musl_based_distro; then
- echo "linux-musl"
- return 0
- else
- echo "linux"
- return 0
- fi
- fi
-
- say_err "OS name could not be detected: UName = $uname"
- return 1
-}
-
-get_legacy_os_name() {
- eval $invocation
-
- local uname=$(uname)
- if [ "$uname" = "Darwin" ]; then
- echo "osx"
- return 0
- elif [ -n "$runtime_id" ]; then
- echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}")
- return 0
- else
- if [ -e /etc/os-release ]; then
- . /etc/os-release
- os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "")
- if [ -n "$os" ]; then
- echo "$os"
- return 0
- fi
- fi
- fi
-
- say_verbose "Distribution specific OS name and version could not be detected: UName = $uname"
- return 1
-}
-
-machine_has() {
- eval $invocation
-
- hash "$1" > /dev/null 2>&1
- return $?
-}
-
-
-check_min_reqs() {
- local hasMinimum=false
- if machine_has "curl"; then
- hasMinimum=true
- elif machine_has "wget"; then
- hasMinimum=true
- fi
-
- if [ "$hasMinimum" = "false" ]; then
- say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed."
- return 1
- fi
- return 0
-}
-
-check_pre_reqs() {
- eval $invocation
-
- if [ "${DOTNET_INSTALL_SKIP_PREREQS:-}" = "1" ]; then
- return 0
- fi
-
- if [ "$(uname)" = "Linux" ]; then
- if is_musl_based_distro; then
- if ! command -v scanelf > /dev/null; then
- say_warning "scanelf not found, please install pax-utils package."
- return 0
- fi
- LDCONFIG_COMMAND="scanelf --ldpath -BF '%f'"
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libintl)" ] && say_warning "Unable to locate libintl. Probable prerequisite missing; install libintl (or gettext)."
- else
- if [ ! -x "$(command -v ldconfig)" ]; then
- say_verbose "ldconfig is not in PATH, trying /sbin/ldconfig."
- LDCONFIG_COMMAND="/sbin/ldconfig"
- else
- LDCONFIG_COMMAND="ldconfig"
- fi
- local librarypath=${LD_LIBRARY_PATH:-}
- LDCONFIG_COMMAND="$LDCONFIG_COMMAND -NXv ${librarypath//:/ }"
- fi
-
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep zlib)" ] && say_warning "Unable to locate zlib. Probable prerequisite missing; install zlib."
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep ssl)" ] && say_warning "Unable to locate libssl. Probable prerequisite missing; install libssl."
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libicu)" ] && say_warning "Unable to locate libicu. Probable prerequisite missing; install libicu."
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep lttng)" ] && say_warning "Unable to locate liblttng. Probable prerequisite missing; install libcurl."
- [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libcurl)" ] && say_warning "Unable to locate libcurl. Probable prerequisite missing; install libcurl."
- fi
-
- return 0
-}
-
-# args:
-# input - $1
-to_lowercase() {
- #eval $invocation
-
- echo "$1" | tr '[:upper:]' '[:lower:]'
- return 0
-}
-
-# args:
-# input - $1
-remove_trailing_slash() {
- #eval $invocation
-
- local input="${1:-}"
- echo "${input%/}"
- return 0
-}
-
-# args:
-# input - $1
-remove_beginning_slash() {
- #eval $invocation
-
- local input="${1:-}"
- echo "${input#/}"
- return 0
-}
-
-# args:
-# root_path - $1
-# child_path - $2 - this parameter can be empty
-combine_paths() {
- eval $invocation
-
- # TODO: Consider making it work with any number of paths. For now:
- if [ ! -z "${3:-}" ]; then
- say_err "combine_paths: Function takes two parameters."
- return 1
- fi
-
- local root_path="$(remove_trailing_slash "$1")"
- local child_path="$(remove_beginning_slash "${2:-}")"
- say_verbose "combine_paths: root_path=$root_path"
- say_verbose "combine_paths: child_path=$child_path"
- echo "$root_path/$child_path"
- return 0
-}
-
-get_machine_architecture() {
- eval $invocation
-
- if command -v uname > /dev/null; then
- CPUName=$(uname -m)
- case $CPUName in
- armv7l)
- echo "arm"
- return 0
- ;;
- aarch64)
- echo "arm64"
- return 0
- ;;
- esac
- fi
-
- # Always default to 'x64'
- echo "x64"
- return 0
-}
-
-# args:
-# architecture - $1
-get_normalized_architecture_from_architecture() {
- eval $invocation
-
- local architecture="$(to_lowercase "$1")"
- case "$architecture" in
- \)
- echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")"
- return 0
- ;;
- amd64|x64)
- echo "x64"
- return 0
- ;;
- arm)
- echo "arm"
- return 0
- ;;
- arm64)
- echo "arm64"
- return 0
- ;;
- esac
-
- say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues"
- return 1
-}
-
-# The version text returned from the feeds is a 1-line or 2-line string:
-# For the SDK and the dotnet runtime (2 lines):
-# Line 1: # commit_hash
-# Line 2: # 4-part version
-# For the aspnetcore runtime (1 line):
-# Line 1: # 4-part version
-
-# args:
-# version_text - stdin
-get_version_from_version_info() {
- eval $invocation
-
- cat | tail -n 1 | sed 's/\r$//'
- return 0
-}
-
-# args:
-# install_root - $1
-# relative_path_to_package - $2
-# specific_version - $3
-is_dotnet_package_installed() {
- eval $invocation
-
- local install_root="$1"
- local relative_path_to_package="$2"
- local specific_version="${3//[$'\t\r\n']}"
-
- local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")"
- say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path"
-
- if [ -d "$dotnet_package_path" ]; then
- return 0
- else
- return 1
- fi
-}
-
-# args:
-# azure_feed - $1
-# channel - $2
-# normalized_architecture - $3
-# coherent - $4
-get_latest_version_info() {
- eval $invocation
-
- local azure_feed="$1"
- local channel="$2"
- local normalized_architecture="$3"
- local coherent="$4"
-
- local version_file_url=null
- if [[ "$runtime" == "dotnet" ]]; then
- version_file_url="$uncached_feed/Runtime/$channel/latest.version"
- elif [[ "$runtime" == "aspnetcore" ]]; then
- version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version"
- elif [ -z "$runtime" ]; then
- if [ "$coherent" = true ]; then
- version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version"
- else
- version_file_url="$uncached_feed/Sdk/$channel/latest.version"
- fi
- else
- say_err "Invalid value for \$runtime"
- return 1
- fi
- say_verbose "get_latest_version_info: latest url: $version_file_url"
-
- download "$version_file_url"
- return $?
-}
-
-# args:
-# json_file - $1
-parse_jsonfile_for_version() {
- eval $invocation
-
- local json_file="$1"
- if [ ! -f "$json_file" ]; then
- say_err "Unable to find \`$json_file\`"
- return 1
- fi
-
- sdk_section=$(cat $json_file | awk '/"sdk"/,/}/')
- if [ -z "$sdk_section" ]; then
- say_err "Unable to parse the SDK node in \`$json_file\`"
- return 1
- fi
-
- sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}')
- sdk_list=${sdk_list//[\" ]/}
- sdk_list=${sdk_list//,/$'\n'}
- sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')"
-
- local version_info=""
- while read -r line; do
- IFS=:
- while read -r key value; do
- if [[ "$key" == "version" ]]; then
- version_info=$value
- fi
- done <<< "$line"
- done <<< "$sdk_list"
- if [ -z "$version_info" ]; then
- say_err "Unable to find the SDK:version node in \`$json_file\`"
- return 1
- fi
-
- unset IFS;
- echo "$version_info"
- return 0
-}
-
-# args:
-# azure_feed - $1
-# channel - $2
-# normalized_architecture - $3
-# version - $4
-# json_file - $5
-get_specific_version_from_version() {
- eval $invocation
-
- local azure_feed="$1"
- local channel="$2"
- local normalized_architecture="$3"
- local version="$(to_lowercase "$4")"
- local json_file="$5"
-
- if [ -z "$json_file" ]; then
- case "$version" in
- latest)
- local version_info
- version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1
- say_verbose "get_specific_version_from_version: version_info=$version_info"
- echo "$version_info" | get_version_from_version_info
- return 0
- ;;
- coherent)
- local version_info
- version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1
- say_verbose "get_specific_version_from_version: version_info=$version_info"
- echo "$version_info" | get_version_from_version_info
- return 0
- ;;
- *)
- echo "$version"
- return 0
- ;;
- esac
- else
- local version_info
- version_info="$(parse_jsonfile_for_version "$json_file")" || return 1
- echo "$version_info"
- return 0
- fi
-}
-
-# args:
-# azure_feed - $1
-# channel - $2
-# normalized_architecture - $3
-# specific_version - $4
-construct_download_link() {
- eval $invocation
-
- local azure_feed="$1"
- local channel="$2"
- local normalized_architecture="$3"
- local specific_version="${4//[$'\t\r\n']}"
- local specific_product_version="$(get_specific_product_version "$1" "$4")"
-
- local osname
- osname="$(get_current_os_name)" || return 1
-
- local download_link=null
- if [[ "$runtime" == "dotnet" ]]; then
- download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz"
- elif [[ "$runtime" == "aspnetcore" ]]; then
- download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz"
- elif [ -z "$runtime" ]; then
- download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz"
- else
- return 1
- fi
-
- echo "$download_link"
- return 0
-}
-
-# args:
-# azure_feed - $1
-# specific_version - $2
-get_specific_product_version() {
- # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents
- # to resolve the version of what's in the folder, superseding the specified version.
- eval $invocation
-
- local azure_feed="$1"
- local specific_version="${2//[$'\t\r\n']}"
- local specific_product_version=$specific_version
-
- local download_link=null
- if [[ "$runtime" == "dotnet" ]]; then
- download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}"
- elif [[ "$runtime" == "aspnetcore" ]]; then
- download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}"
- elif [ -z "$runtime" ]; then
- download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}"
- else
- return 1
- fi
-
- specific_product_version=$(curl -s --fail "$download_link")
- if [ $? -ne 0 ]
- then
- specific_product_version=$(wget -qO- "$download_link")
- if [ $? -ne 0 ]
- then
- specific_product_version=$specific_version
- fi
- fi
- specific_product_version="${specific_product_version//[$'\t\r\n']}"
-
- echo "$specific_product_version"
- return 0
-}
-
-# args:
-# azure_feed - $1
-# channel - $2
-# normalized_architecture - $3
-# specific_version - $4
-construct_legacy_download_link() {
- eval $invocation
-
- local azure_feed="$1"
- local channel="$2"
- local normalized_architecture="$3"
- local specific_version="${4//[$'\t\r\n']}"
-
- local distro_specific_osname
- distro_specific_osname="$(get_legacy_os_name)" || return 1
-
- local legacy_download_link=null
- if [[ "$runtime" == "dotnet" ]]; then
- legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz"
- elif [ -z "$runtime" ]; then
- legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz"
- else
- return 1
- fi
-
- echo "$legacy_download_link"
- return 0
-}
-
-get_user_install_path() {
- eval $invocation
-
- if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then
- echo "$DOTNET_INSTALL_DIR"
- else
- echo "$HOME/.dotnet"
- fi
- return 0
-}
-
-# args:
-# install_dir - $1
-resolve_installation_path() {
- eval $invocation
-
- local install_dir=$1
- if [ "$install_dir" = "" ]; then
- local user_install_path="$(get_user_install_path)"
- say_verbose "resolve_installation_path: user_install_path=$user_install_path"
- echo "$user_install_path"
- return 0
- fi
-
- echo "$install_dir"
- return 0
-}
-
-# args:
-# relative_or_absolute_path - $1
-get_absolute_path() {
- eval $invocation
-
- local relative_or_absolute_path=$1
- echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")"
- return 0
-}
-
-# args:
-# input_files - stdin
-# root_path - $1
-# out_path - $2
-# override - $3
-copy_files_or_dirs_from_list() {
- eval $invocation
-
- local root_path="$(remove_trailing_slash "$1")"
- local out_path="$(remove_trailing_slash "$2")"
- local override="$3"
- local osname="$(get_current_os_name)"
- local override_switch=$(
- if [ "$override" = false ]; then
- if [ "$osname" = "linux-musl" ]; then
- printf -- "-u";
- else
- printf -- "-n";
- fi
- fi)
-
- cat | uniq | while read -r file_path; do
- local path="$(remove_beginning_slash "${file_path#$root_path}")"
- local target="$out_path/$path"
- if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then
- mkdir -p "$out_path/$(dirname "$path")"
- if [ -d "$target" ]; then
- rm -rf "$target"
- fi
- cp -R $override_switch "$root_path/$path" "$target"
- fi
- done
-}
-
-# args:
-# zip_path - $1
-# out_path - $2
-extract_dotnet_package() {
- eval $invocation
-
- local zip_path="$1"
- local out_path="$2"
-
- local temp_out_path="$(mktemp -d "$temporary_file_template")"
-
- local failed=false
- tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true
-
- local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/'
- find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false
- find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files"
-
- rm -rf "$temp_out_path"
-
- if [ "$failed" = true ]; then
- say_err "Extraction failed"
- return 1
- fi
-}
-
-# args:
-# remote_path - $1
-# [out_path] - $2 - stdout if not provided
-download() {
- eval $invocation
-
- local remote_path="$1"
- local out_path="${2:-}"
-
- if [[ "$remote_path" != "http"* ]]; then
- cp "$remote_path" "$out_path"
- return $?
- fi
-
- local failed=false
- if machine_has "curl"; then
- downloadcurl "$remote_path" "$out_path" || failed=true
- elif machine_has "wget"; then
- downloadwget "$remote_path" "$out_path" || failed=true
- else
- failed=true
- fi
- if [ "$failed" = true ]; then
- say_verbose "Download failed: $remote_path"
- return 1
- fi
- return 0
-}
-
-downloadcurl() {
- eval $invocation
- local remote_path="$1"
- local out_path="${2:-}"
-
- # Append feed_credential as late as possible before calling curl to avoid logging feed_credential
- remote_path="${remote_path}${feed_credential}"
-
- local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs "
- local failed=false
- if [ -z "$out_path" ]; then
- curl $curl_options "$remote_path" || failed=true
- else
- curl $curl_options -o "$out_path" "$remote_path" || failed=true
- fi
- if [ "$failed" = true ]; then
- say_verbose "Curl download failed"
- return 1
- fi
- return 0
-}
-
-downloadwget() {
- eval $invocation
- local remote_path="$1"
- local out_path="${2:-}"
-
- # Append feed_credential as late as possible before calling wget to avoid logging feed_credential
- remote_path="${remote_path}${feed_credential}"
- local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 "
- local failed=false
- if [ -z "$out_path" ]; then
- wget -q $wget_options -O - "$remote_path" || failed=true
- else
- wget $wget_options -O "$out_path" "$remote_path" || failed=true
- fi
- if [ "$failed" = true ]; then
- say_verbose "Wget download failed"
- return 1
- fi
- return 0
-}
-
-calculate_vars() {
- eval $invocation
- valid_legacy_download_link=true
-
- normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")"
- say_verbose "normalized_architecture=$normalized_architecture"
-
- specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")"
- specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")"
- say_verbose "specific_version=$specific_version"
- if [ -z "$specific_version" ]; then
- say_err "Could not resolve version information."
- return 1
- fi
-
- download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")"
- say_verbose "Constructed primary named payload URL: $download_link"
-
- legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false
-
- if [ "$valid_legacy_download_link" = true ]; then
- say_verbose "Constructed legacy named payload URL: $legacy_download_link"
- else
- say_verbose "Cound not construct a legacy_download_link; omitting..."
- fi
-
- install_root="$(resolve_installation_path "$install_dir")"
- say_verbose "InstallRoot: $install_root"
-}
-
-install_dotnet() {
- eval $invocation
- local download_failed=false
- local asset_name=''
- local asset_relative_path=''
-
- if [[ "$runtime" == "dotnet" ]]; then
- asset_relative_path="shared/Microsoft.NETCore.App"
- asset_name=".NET Core Runtime"
- elif [[ "$runtime" == "aspnetcore" ]]; then
- asset_relative_path="shared/Microsoft.AspNetCore.App"
- asset_name="ASP.NET Core Runtime"
- elif [ -z "$runtime" ]; then
- asset_relative_path="sdk"
- asset_name=".NET Core SDK"
- else
- say_err "Invalid value for \$runtime"
- return 1
- fi
-
- # Check if the SDK version is already installed.
- if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then
- say "$asset_name version $specific_version is already installed."
- return 0
- fi
-
- mkdir -p "$install_root"
- zip_path="$(mktemp "$temporary_file_template")"
- say_verbose "Zip path: $zip_path"
-
- say "Downloading link: $download_link"
-
- # Failures are normal in the non-legacy case for ultimately legacy downloads.
- # Do not output to stderr, since output to stderr is considered an error.
- download "$download_link" "$zip_path" 2>&1 || download_failed=true
-
- # if the download fails, download the legacy_download_link
- if [ "$download_failed" = true ]; then
- say "Cannot download: $download_link"
-
- if [ "$valid_legacy_download_link" = true ]; then
- download_failed=false
- download_link="$legacy_download_link"
- zip_path="$(mktemp "$temporary_file_template")"
- say_verbose "Legacy zip path: $zip_path"
- say "Downloading legacy link: $download_link"
- download "$download_link" "$zip_path" 2>&1 || download_failed=true
-
- if [ "$download_failed" = true ]; then
- say "Cannot download: $download_link"
- fi
- fi
- fi
-
- if [ "$download_failed" = true ]; then
- say_err "Could not find/download: \`$asset_name\` with version = $specific_version"
- say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
- return 1
- fi
-
- say "Extracting zip from $download_link"
- extract_dotnet_package "$zip_path" "$install_root"
-
- # Check if the SDK version is installed; if not, fail the installation.
- # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed.
- if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then
- IFS='-'
- read -ra verArr <<< "$specific_version"
- release_version="${verArr[0]}"
- unset IFS;
- say_verbose "Checking installation: version = $release_version"
- if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then
- return 0
- fi
- fi
-
- # Check if the standard SDK version is installed.
- say_verbose "Checking installation: version = $specific_product_version"
- if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then
- return 0
- fi
-
- say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error."
- return 1
-}
-
-args=("$@")
-
-local_version_file_relative_path="/.version"
-bin_folder_relative_path=""
-temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX"
-
-channel="LTS"
-version="Latest"
-json_file=""
-install_dir=""
-architecture=""
-dry_run=false
-no_path=false
-no_cdn=false
-azure_feed="https://dotnetcli.azureedge.net/dotnet"
-uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet"
-feed_credential=""
-verbose=false
-runtime=""
-runtime_id=""
-override_non_versioned_files=true
-non_dynamic_parameters=""
-
-while [ $# -ne 0 ]
-do
- name="$1"
- case "$name" in
- -c|--channel|-[Cc]hannel)
- shift
- channel="$1"
- ;;
- -v|--version|-[Vv]ersion)
- shift
- version="$1"
- ;;
- -i|--install-dir|-[Ii]nstall[Dd]ir)
- shift
- install_dir="$1"
- ;;
- --arch|--architecture|-[Aa]rch|-[Aa]rchitecture)
- shift
- architecture="$1"
- ;;
- --shared-runtime|-[Ss]hared[Rr]untime)
- say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'."
- if [ -z "$runtime" ]; then
- runtime="dotnet"
- fi
- ;;
- --runtime|-[Rr]untime)
- shift
- runtime="$1"
- if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then
- say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'."
- if [[ "$runtime" == "windowsdesktop" ]]; then
- say_err "WindowsDesktop archives are manufactured for Windows platforms only."
- fi
- exit 1
- fi
- ;;
- --dry-run|-[Dd]ry[Rr]un)
- dry_run=true
- ;;
- --no-path|-[Nn]o[Pp]ath)
- no_path=true
- non_dynamic_parameters+=" $name"
- ;;
- --verbose|-[Vv]erbose)
- verbose=true
- non_dynamic_parameters+=" $name"
- ;;
- --no-cdn|-[Nn]o[Cc]dn)
- no_cdn=true
- non_dynamic_parameters+=" $name"
- ;;
- --azure-feed|-[Aa]zure[Ff]eed)
- shift
- azure_feed="$1"
- non_dynamic_parameters+=" $name "\""$1"\"""
- ;;
- --uncached-feed|-[Uu]ncached[Ff]eed)
- shift
- uncached_feed="$1"
- non_dynamic_parameters+=" $name "\""$1"\"""
- ;;
- --feed-credential|-[Ff]eed[Cc]redential)
- shift
- feed_credential="$1"
- non_dynamic_parameters+=" $name "\""$1"\"""
- ;;
- --runtime-id|-[Rr]untime[Ii]d)
- shift
- runtime_id="$1"
- non_dynamic_parameters+=" $name "\""$1"\"""
- ;;
- --jsonfile|-[Jj][Ss]on[Ff]ile)
- shift
- json_file="$1"
- ;;
- --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles)
- override_non_versioned_files=false
- non_dynamic_parameters+=" $name"
- ;;
- -?|--?|-h|--help|-[Hh]elp)
- script_name="$(basename "$0")"
- echo ".NET Tools Installer"
- echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]"
- echo " $script_name -h|-?|--help"
- echo ""
- echo "$script_name is a simple command line interface for obtaining dotnet cli."
- echo ""
- echo "Options:"
- echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`."
- echo " -Channel"
- echo " Possible values:"
- echo " - Current - most current release"
- echo " - LTS - most current supported release"
- echo " - 2-part version in a format A.B - represents a specific release"
- echo " examples: 2.0; 1.0"
- echo " - Branch name"
- echo " examples: release/2.0.0; Master"
- echo " Note: The version parameter overrides the channel parameter."
- echo " -v,--version Use specific VERSION, Defaults to \`$version\`."
- echo " -Version"
- echo " Possible values:"
- echo " - latest - most latest build on specific channel"
- echo " - coherent - most latest coherent build on specific channel"
- echo " coherent applies only to SDK downloads"
- echo " - 3-part version in a format A.B.C - represents specific version of build"
- echo " examples: 2.0.0-preview2-006120; 1.1.0"
- echo " -i,--install-dir Install under specified location (see Install Location below)"
- echo " -InstallDir"
- echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`."
- echo " --arch,-Architecture,-Arch"
- echo " Possible values: x64, arm, and arm64"
- echo " --runtime Installs a shared runtime only, without the SDK."
- echo " -Runtime"
- echo " Possible values:"
- echo " - dotnet - the Microsoft.NETCore.App shared runtime"
- echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime"
- echo " --dry-run,-DryRun Do not perform installation. Display download link."
- echo " --no-path, -NoPath Do not set PATH for the current process."
- echo " --verbose,-Verbose Display diagnostics information."
- echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user."
- echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user."
- echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified."
- echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable."
- echo " -SkipNonVersionedFiles"
- echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly."
- echo " --jsonfile Determines the SDK version from a user specified global.json file."
- echo " Note: global.json must have a value for 'SDK:Version'"
- echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)."
- echo " -RuntimeId"
- echo " -?,--?,-h,--help,-Help Shows this help message"
- echo ""
- echo "Obsolete parameters:"
- echo " --shared-runtime The recommended alternative is '--runtime dotnet'."
- echo " This parameter is obsolete and may be removed in a future version of this script."
- echo " Installs just the shared runtime bits, not the entire SDK."
- echo ""
- echo "Install Location:"
- echo " Location is chosen in following order:"
- echo " - --install-dir option"
- echo " - Environmental variable DOTNET_INSTALL_DIR"
- echo " - $HOME/.dotnet"
- exit 0
- ;;
- *)
- say_err "Unknown argument \`$name\`"
- exit 1
- ;;
- esac
-
- shift
-done
-
-if [ "$no_cdn" = true ]; then
- azure_feed="$uncached_feed"
-fi
-
-check_min_reqs
-calculate_vars
-script_name=$(basename "$0")
-
-if [ "$dry_run" = true ]; then
- say "Payload URLs:"
- say "Primary named payload URL: $download_link"
- if [ "$valid_legacy_download_link" = true ]; then
- say "Legacy named payload URL: $legacy_download_link"
- fi
- repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"""
- if [[ "$runtime" == "dotnet" ]]; then
- repeatable_command+=" --runtime "\""dotnet"\"""
- elif [[ "$runtime" == "aspnetcore" ]]; then
- repeatable_command+=" --runtime "\""aspnetcore"\"""
- fi
- repeatable_command+="$non_dynamic_parameters"
- say "Repeatable invocation: $repeatable_command"
- exit 0
-fi
-
-check_pre_reqs
-install_dotnet
-
-bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")"
-if [ "$no_path" = false ]; then
- say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script."
- export PATH="$bin_path":"$PATH"
-else
- say "Binaries of dotnet can be found in $bin_path"
-fi
-
-say "Installation finished successfully."
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
index ead6a1d9a2..d6efeb4434 100755
--- a/eng/common/dotnet-install.sh
+++ b/eng/common/dotnet-install.sh
@@ -19,7 +19,7 @@ runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)
shift
@@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
shift
done
-# Use uname to determine what the CPU is.
-cpuname=$(uname -p)
-# Some Linux platforms report unknown for platform, but the arch for machine.
-if [[ "$cpuname" == "unknown" ]]; then
- cpuname=$(uname -m)
-fi
-
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
case $cpuname in
aarch64)
buildarch=arm64
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
new file mode 100644
index 0000000000..24c00b5be9
--- /dev/null
+++ b/eng/common/generate-locproject.ps1
@@ -0,0 +1,110 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
+ [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
+ [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+ [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+ $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+ $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+ $firstLangCode = $Matches.1
+ $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+ $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
+
+ $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+ $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+ Projects = @(
+ @{
+ LanguageSet = $LanguageSet
+ LocItems = @(
+ $locFiles | ForEach-Object {
+ $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($outputPath.Contains($exclusion))
+ {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+ Remove-Item -Path $sourceFile
+ }
+ if ($continue)
+ {
+ if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
+ }
+ }
+ else {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnName"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ }
+ )
+ }
+ )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+ New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+}
+else {
+ New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+ Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+
+ if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+ Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+
+ exit 1
+ }
+ else {
+ Write-Host "Generated LocProject.json and current LocProject.json are identical."
+ }
+}
\ No newline at end of file
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
index 29fc5db8ae..5bd205b5da 100755
--- a/eng/common/init-tools-native.sh
+++ b/eng/common/init-tools-native.sh
@@ -16,7 +16,7 @@ declare -A native_assets
. $scriptroot/native/common-library.sh
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -76,24 +76,89 @@ while (($# > 0)); do
done
function ReadGlobalJsonNativeTools {
- # Get the native-tools section from the global.json.
- local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
- # Only extract the contents of the object.
- local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
- native_tools_list=${native_tools_list//[\" ]/}
- native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-
- local old_IFS=$IFS
- while read -r line; do
- # Lines are of the form: 'tool:version'
- IFS=:
- while read -r key value; do
- native_assets[$key]=$value
- done <<< "$line"
- done <<< "$native_tools_list"
- IFS=$old_IFS
-
- return 0;
+ # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+ if command -v jq &> /dev/null; then
+
+ # jq: read each key/value pair under "native-tools" entry and emit:
+ # KEY="" VALUE=""
+ # followed by a null byte.
+ #
+ # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+ while IFS= read -rd '' line; do
+ native_assets+=("$line")
+ done < <(jq -r '. |
+ select(has("native-tools")) |
+ ."native-tools" |
+ keys[] as $k |
+ @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+ return
+ fi
+
+ # Warning: falling back to manually parsing JSON, which is not recommended.
+
+ # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+ # It has been tested with several weird strings with escaped characters in entries (key and value)
+ # and results were compared with the output of jq(1) in binary representation using xxd(1);
+ # just before the assignment to 'native_assets' array (above and below).
+
+ # try to capture the section under "native-tools".
+ if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+ return
+ fi
+
+ section="${BASH_REMATCH[1]}"
+
+ parseStarted=0
+ possibleEnd=0
+ escaping=0
+ escaped=0
+ isKey=1
+
+ for (( i=0; i<${#section}; i++ )); do
+ char="${section:$i:1}"
+ if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+ if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+ escaping=1
+ elif ((escaping)) && ! ((escaped)); then
+ escaped=1
+ fi
+
+ if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+ parseStarted=1
+ possibleEnd=0
+ elif [[ "$char" == "'" ]]; then
+ token="$token'\\\''"
+ possibleEnd=0
+ elif ((escaping)) || [[ "$char" != "\"" ]]; then
+ token="$token$char"
+ possibleEnd=1
+ fi
+
+ if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+ # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+ # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+ printf -v token "'$token'"
+
+ if ((isKey)); then
+ KEY="$token"
+ isKey=0
+ else
+ line="KEY=$KEY VALUE=$token"
+ native_assets+=("$line")
+ isKey=1
+ fi
+
+ # reset for next token
+ parseStarted=0
+ token=
+ elif ((escaping)) && ((escaped)); then
+ escaping=0
+ escaped=0
+ fi
+ done
}
native_base_dir=$install_directory
@@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
exit 0;
else
native_installer_dir="$scriptroot/native"
- for tool in "${!native_assets[@]}"
- do
- tool_version=${native_assets[$tool]}
- installer_path="$native_installer_dir/install-$tool.sh"
+ for index in "${!native_assets[@]}"; do
+ eval "${native_assets["$index"]}"
+
+ installer_path="$native_installer_dir/install-$KEY.sh"
installer_command="$installer_path"
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
- installer_command+=" --version $tool_version"
+ installer_command+=" --version $VALUE"
echo $installer_command
if [[ $force = true ]]; then
diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
index b8f6529fdc..418c09930c 100644
--- a/eng/common/internal-feed-operations.ps1
+++ b/eng/common/internal-feed-operations.ps1
@@ -63,8 +63,6 @@ function SetupCredProvider {
}
if (($endpoints | Measure-Object).Count -gt 0) {
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
$endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
# Create the environment variables the AzDo way
diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
index 9ed225e7e5..e2233e7812 100755
--- a/eng/common/internal-feed-operations.sh
+++ b/eng/common/internal-feed-operations.sh
@@ -62,8 +62,6 @@ function SetupCredProvider {
endpoints+=']'
if [ ${#endpoints} -gt 2 ]; then
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@@ -103,7 +101,7 @@ authToken=''
repoName=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)
operation=$2
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
index 8160cd5a59..20d3dad543 100755
--- a/eng/common/msbuild.sh
+++ b/eng/common/msbuild.sh
@@ -19,7 +19,7 @@ prepare_machine=false
extra_args=''
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--verbosity)
verbosity=$2
diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1
index d7d1a65109..adf707c8fe 100644
--- a/eng/common/native/CommonLibrary.psm1
+++ b/eng/common/native/CommonLibrary.psm1
@@ -48,7 +48,7 @@ function DownloadAndExtract {
-Verbose:$Verbose
if ($DownloadStatus -Eq $False) {
- Write-Error "Download failed"
+ Write-Error "Download failed from $Uri"
return $False
}
diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
index 12339a4076..8a5e7cf0db 100755
--- a/eng/common/native/install-cmake-test.sh
+++ b/eng/common/native/install-cmake-test.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake-test"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
index 18041be876..de496beebc 100755
--- a/eng/common/native/install-cmake.sh
+++ b/eng/common/native/install-cmake.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1
index f397e1c75d..78f2d84a4e 100644
--- a/eng/common/native/install-tool.ps1
+++ b/eng/common/native/install-tool.ps1
@@ -105,7 +105,7 @@ try {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1
} elseif (@($ToolFilePath).Length -Lt 1) {
- Write-Host "$ToolName was not found in $ToolFilePath."
+ Write-Host "$ToolName was not found in $ToolInstallDirectory."
exit 1
}
diff --git a/eng/common/performance/blazor_perf.proj b/eng/common/performance/blazor_perf.proj
deleted file mode 100644
index 3b25359c43..0000000000
--- a/eng/common/performance/blazor_perf.proj
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
- python3
- $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk
-
-
-
-
- %(Identity)
-
-
-
-
- %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\
- $(ScenarioDirectory)blazor\
-
-
- $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/
- $(ScenarioDirectory)blazor/
-
-
-
-
- $(WorkItemDirectory)
- cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27
- $(Python) test.py sod --scenario-name "%(Identity)"
- $(Python) post.py
-
-
-
\ No newline at end of file
diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj
deleted file mode 100644
index 4264920382..0000000000
--- a/eng/common/performance/crossgen_perf.proj
+++ /dev/null
@@ -1,69 +0,0 @@
-
-
-
-
- %(Identity)
-
-
-
-
-
- py -3
- $(HelixPreCommands)
- %HELIX_CORRELATION_PAYLOAD%\Core_Root
- %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\
- $(ScenarioDirectory)crossgen\
- $(ScenarioDirectory)crossgen2\
-
-
- python3
- $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update
- $HELIX_CORRELATION_PAYLOAD/Core_Root
- $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/
- $(ScenarioDirectory)crossgen/
- $(ScenarioDirectory)crossgen2/
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- $(WorkItemDirectory)
- $(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity)
-
-
-
-
-
- $(WorkItemDirectory)
- $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity)
-
-
-
-
-
-
- 4:00
-
-
-
- 4:00
-
-
- $(WorkItemDirectory)
- $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp
- 1:00
-
-
-
\ No newline at end of file
diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj
deleted file mode 100644
index 94b6efbc92..0000000000
--- a/eng/common/performance/microbenchmarks.proj
+++ /dev/null
@@ -1,144 +0,0 @@
-
-
-
- %HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj)
- --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%
- py -3
- %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe
- %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe
-
- $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%
- %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts
- %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline
- %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj
- %HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe
- %25%25
- %HELIX_WORKITEM_ROOT%\testResults.xml
-
-
-
- $HELIX_CORRELATION_PAYLOAD
- $(BaseDirectory)/performance
-
-
-
- $HELIX_WORKITEM_PAYLOAD
- $(BaseDirectory)
-
-
-
- $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)
- --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP
- python3
- $(BaseDirectory)/Core_Root/corerun
- $(BaseDirectory)/Baseline_Core_Root/corerun
- $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh
- $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts
- $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline
- $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj
- $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet
- %25
- $HELIX_WORKITEM_ROOT/testResults.xml
-
-
-
- $(CliArguments) --wasm
-
-
-
- --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe
-
-
- --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun
-
-
-
- --corerun $(CoreRun)
-
-
-
- --corerun $(BaselineCoreRun)
-
-
-
- $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)
-
-
-
- $(WorkItemCommand) $(CliArguments)
-
-
-
- 2:30
- 0:15
-
-
-
-
- %(Identity)
-
-
-
-
- 30
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- false
-
-
-
-
-
- $(WorkItemDirectory)
- $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"
- $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"
- $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)
- $(WorkItemTimeout)
-
-
-
-
-
- $(WorkItemDirectory)
- $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"
- $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"
- $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)
- 4:00
-
-
-
\ No newline at end of file
diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1
deleted file mode 100644
index 656c0bd902..0000000000
--- a/eng/common/performance/performance-setup.ps1
+++ /dev/null
@@ -1,147 +0,0 @@
-Param(
- [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
- [string] $CoreRootDirectory,
- [string] $BaselineCoreRootDirectory,
- [string] $Architecture="x64",
- [string] $Framework="net5.0",
- [string] $CompilationMode="Tiered",
- [string] $Repository=$env:BUILD_REPOSITORY_NAME,
- [string] $Branch=$env:BUILD_SOURCEBRANCH,
- [string] $CommitSha=$env:BUILD_SOURCEVERSION,
- [string] $BuildNumber=$env:BUILD_BUILDNUMBER,
- [string] $RunCategories="Libraries Runtime",
- [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
- [string] $Kind="micro",
- [switch] $LLVM,
- [switch] $MonoInterpreter,
- [switch] $MonoAOT,
- [switch] $Internal,
- [switch] $Compare,
- [string] $MonoDotnet="",
- [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
-)
-
-$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
-$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
-$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
-
-$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
-$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
-$WorkItemDirectory = (Join-Path $SourceDirectory "workitem")
-$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
-$Creator = $env:BUILD_DEFINITIONNAME
-$PerfLabArguments = ""
-$HelixSourcePrefix = "pr"
-
-$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
-
-# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5.
-if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) {
- $Queue = "Windows.10.Amd64.ClientRS5.Open"
-}
-
-if ($Compare) {
- $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
- $PerfLabArguments = ""
- $ExtraBenchmarkDotNetArguments = ""
-}
-
-if ($Internal) {
- $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
- $PerfLabArguments = "--upload-to-perflab-container"
- $ExtraBenchmarkDotNetArguments = ""
- $Creator = ""
- $HelixSourcePrefix = "official"
-}
-
-if($MonoInterpreter)
-{
- $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter"
-}
-
-if($MonoDotnet -ne "")
-{
- $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT"
- if($ExtraBenchmarkDotNetArguments -eq "")
- {
- #FIX ME: We need to block these tests as they don't run on mono for now
- $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*"
- }
- else
- {
- #FIX ME: We need to block these tests as they don't run on mono for now
- $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*"
- }
-}
-
-# FIX ME: This is a workaround until we get this from the actual pipeline
-$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture"
-$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
-
-
-#This grabs the LKG version number of dotnet and passes it to our scripts
-$VersionJSON = Get-Content global.json | ConvertFrom-Json
-$DotNetVersion = $VersionJSON.tools.dotnet
-$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments"
-
-
-if ($RunFromPerformanceRepo) {
- $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
-
- robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
-}
-else {
- git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
-}
-
-if($MonoDotnet -ne "")
-{
- $UsingMono = "true"
- $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono")
- Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath
-}
-
-if ($UseCoreRun) {
- $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
- Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
-}
-if ($UseBaselineCoreRun) {
- $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
- Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
-}
-
-$DocsDir = (Join-Path $PerformanceDirectory "docs")
-robocopy $DocsDir $WorkItemDirectory
-
-# Set variables that we will need to have in future steps
-$ci = $true
-
-. "$PSScriptRoot\..\pipeline-logging-functions.ps1"
-
-# Directories
-Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false
-
-# Script Arguments
-Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false
-
-# Helix Arguments
-Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false
-Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false
-
-exit 0
\ No newline at end of file
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
deleted file mode 100755
index 806e56c612..0000000000
--- a/eng/common/performance/performance-setup.sh
+++ /dev/null
@@ -1,289 +0,0 @@
-#!/usr/bin/env bash
-
-source_directory=$BUILD_SOURCESDIRECTORY
-core_root_directory=
-baseline_core_root_directory=
-architecture=x64
-framework=net5.0
-compilation_mode=tiered
-repository=$BUILD_REPOSITORY_NAME
-branch=$BUILD_SOURCEBRANCH
-commit_sha=$BUILD_SOURCEVERSION
-build_number=$BUILD_BUILDNUMBER
-internal=false
-compare=false
-mono_dotnet=
-kind="micro"
-llvm=false
-monointerpreter=false
-monoaot=false
-run_categories="Libraries Runtime"
-csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
-configurations="CompliationMode=$compilation_mode RunKind=$kind"
-run_from_perf_repo=false
-use_core_run=true
-use_baseline_core_run=true
-using_mono=false
-wasm_runtime_loc=
-using_wasm=false
-use_latest_dotnet=false
-
-while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
- case $lowerI in
- --sourcedirectory)
- source_directory=$2
- shift 2
- ;;
- --corerootdirectory)
- core_root_directory=$2
- shift 2
- ;;
- --baselinecorerootdirectory)
- baseline_core_root_directory=$2
- shift 2
- ;;
- --architecture)
- architecture=$2
- shift 2
- ;;
- --framework)
- framework=$2
- shift 2
- ;;
- --compilationmode)
- compilation_mode=$2
- shift 2
- ;;
- --repository)
- repository=$2
- shift 2
- ;;
- --branch)
- branch=$2
- shift 2
- ;;
- --commitsha)
- commit_sha=$2
- shift 2
- ;;
- --buildnumber)
- build_number=$2
- shift 2
- ;;
- --kind)
- kind=$2
- configurations="CompilationMode=$compilation_mode RunKind=$kind"
- shift 2
- ;;
- --runcategories)
- run_categories=$2
- shift 2
- ;;
- --csproj)
- csproj=$2
- shift 2
- ;;
- --internal)
- internal=true
- shift 1
- ;;
- --llvm)
- llvm=true
- shift 1
- ;;
- --monointerpreter)
- monointerpreter=true
- shift 1
- ;;
- --monoaot)
- monoaot=true
- shift 1
- ;;
- --monodotnet)
- mono_dotnet=$2
- shift 2
- ;;
- --wasm)
- wasm_runtime_loc=$2
- shift 2
- ;;
- --compare)
- compare=true
- shift 1
- ;;
- --configurations)
- configurations=$2
- shift 2
- ;;
- --latestdotnet)
- use_latest_dotnet=true
- shift 1
- ;;
- *)
- echo "Common settings:"
- echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun"
- echo " --architecture Architecture of the testing being run"
- echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure."
- echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\""
- echo " --help Print help and exit"
- echo ""
- echo "Advanced settings:"
- echo " --framework The framework to run, if not running in master"
- echo " --compliationmode The compilation mode if not passing --configurations"
- echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY"
- echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME"
- echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH"
- echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION"
- echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER"
- echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj"
- echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro"
- echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\""
- echo " --internal If the benchmarks are running as an official job."
- echo " --monodotnet Pass the path to the mono dotnet for mono performance testing."
- echo " --wasm Path to the unpacked wasm runtime pack."
- echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json "
- echo ""
- exit 0
- ;;
- esac
-done
-
-if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then
- run_from_perf_repo=true
-fi
-
-if [ -z "$configurations" ]; then
- configurations="CompilationMode=$compilation_mode"
-fi
-
-if [ -z "$core_root_directory" ]; then
- use_core_run=false
-fi
-
-if [ -z "$baseline_core_root_directory" ]; then
- use_baseline_core_run=false
-fi
-
-payload_directory=$source_directory/Payload
-performance_directory=$payload_directory/performance
-workitem_directory=$source_directory/workitem
-extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
-perflab_arguments=
-queue=Ubuntu.1804.Amd64.Open
-creator=$BUILD_DEFINITIONNAME
-helix_source_prefix="pr"
-
-if [[ "$compare" == true ]]; then
- extra_benchmark_dotnet_arguments=
- perflab_arguments=
-
- # No open queues for arm64
- if [[ "$architecture" = "arm64" ]]; then
- echo "Compare not available for arm64"
- exit 1
- fi
-
- queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
-fi
-
-if [[ "$internal" == true ]]; then
- perflab_arguments="--upload-to-perflab-container"
- helix_source_prefix="official"
- creator=
- extra_benchmark_dotnet_arguments=
-
- if [[ "$architecture" = "arm64" ]]; then
- queue=Ubuntu.1804.Arm64.Perf
- else
- queue=Ubuntu.1804.Amd64.Tiger.Perf
- fi
-fi
-
-if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
- configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
-fi
-
-if [[ "$wasm_runtime_loc" != "" ]]; then
- configurations="CompilationMode=wasm RunKind=$kind"
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono"
-fi
-
-if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
-fi
-
-common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
-setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
-
-
-if [[ "$use_latest_dotnet" = false ]]; then
- # Get the tools section from the global.json.
- # This grabs the LKG version number of dotnet and passes it to our scripts
- dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'`
- setup_arguments="--dotnet-versions $dotnet_version $setup_arguments"
-fi
-
-if [[ "$run_from_perf_repo" = true ]]; then
- payload_directory=
- workitem_directory=$source_directory
- performance_directory=$workitem_directory
- setup_arguments="--perf-hash $commit_sha $common_setup_arguments"
-else
- git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
-
- docs_directory=$performance_directory/docs
- mv $docs_directory $workitem_directory
-fi
-
-if [[ "$wasm_runtime_loc" != "" ]]; then
- using_wasm=true
- wasm_dotnet_path=$payload_directory/dotnet-wasm
- mv $wasm_runtime_loc $wasm_dotnet_path
- extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
-fi
-
-if [[ "$mono_dotnet" != "" ]]; then
- using_mono=true
- mono_dotnet_path=$payload_directory/dotnet-mono
- mv $mono_dotnet $mono_dotnet_path
-fi
-
-if [[ "$use_core_run" = true ]]; then
- new_core_root=$payload_directory/Core_Root
- mv $core_root_directory $new_core_root
-fi
-
-if [[ "$use_baseline_core_run" = true ]]; then
- new_baseline_core_root=$payload_directory/Baseline_Core_Root
- mv $baseline_core_root_directory $new_baseline_core_root
-fi
-
-ci=true
-
-_script_dir=$(pwd)/eng/common
-. "$_script_dir/pipeline-logging-functions.sh"
-
-# Make sure all of our variables are available for future steps
-Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
-Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
-Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
-Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
-Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
-Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false
-Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
-Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
-Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
-Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
-Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
-Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
-Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
-Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
-Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false
-Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false
diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1
index 8484451f3a..8e422c561e 100644
--- a/eng/common/pipeline-logging-functions.ps1
+++ b/eng/common/pipeline-logging-functions.ps1
@@ -29,14 +29,14 @@ function Write-PipelineTelemetryError {
[switch]$AsOutput,
[switch]$Force)
- $PSBoundParameters.Remove('Category') | Out-Null
+ $PSBoundParameters.Remove('Category') | Out-Null
- if($Force -Or ((Test-Path variable:ci) -And $ci)) {
- $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
- }
- $PSBoundParameters.Remove('Message') | Out-Null
- $PSBoundParameters.Add('Message', $Message)
- Write-PipelineTaskError @PSBoundParameters
+ if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+ $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+ }
+ $PSBoundParameters.Remove('Message') | Out-Null
+ $PSBoundParameters.Add('Message', $Message)
+ Write-PipelineTaskError @PSBoundParameters
}
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
@@ -55,8 +55,8 @@ function Write-PipelineTaskError {
[switch]$Force
)
- if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
- if($Type -eq 'error') {
+ if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+ if ($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red
return
}
@@ -66,47 +66,61 @@ function Write-PipelineTaskError {
}
}
- if(($Type -ne 'error') -and ($Type -ne 'warning')) {
+ if (($Type -ne 'error') -and ($Type -ne 'warning')) {
Write-Host $Message
return
}
$PSBoundParameters.Remove('Force') | Out-Null
- if(-not $PSBoundParameters.ContainsKey('Type')) {
+ if (-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error')
}
Write-LogIssue @PSBoundParameters
- }
+}
- function Write-PipelineSetVariable {
+function Write-PipelineSetVariable {
[CmdletBinding()]
param(
- [Parameter(Mandatory = $true)]
- [string]$Name,
- [string]$Value,
- [switch]$Secret,
- [switch]$AsOutput,
- [bool]$IsMultiJobVariable=$true)
-
- if((Test-Path variable:ci) -And $ci) {
+ [Parameter(Mandatory = $true)]
+ [string]$Name,
+ [string]$Value,
+ [switch]$Secret,
+ [switch]$AsOutput,
+ [bool]$IsMultiJobVariable = $true)
+
+ if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
- 'variable' = $Name
- 'isSecret' = $Secret
- 'isOutput' = $IsMultiJobVariable
+ 'variable' = $Name
+ 'isSecret' = $Secret
+ 'isOutput' = $IsMultiJobVariable
} -AsOutput:$AsOutput
- }
- }
+ }
+}
- function Write-PipelinePrependPath {
+function Write-PipelinePrependPath {
[CmdletBinding()]
param(
- [Parameter(Mandatory=$true)]
- [string]$Path,
- [switch]$AsOutput)
+ [Parameter(Mandatory = $true)]
+ [string]$Path,
+ [switch]$AsOutput)
- if((Test-Path variable:ci) -And $ci) {
+ if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
- }
- }
+ }
+}
+
+function Write-PipelineSetResult {
+ [CmdletBinding()]
+ param(
+ [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+ [Parameter(Mandatory = $true)]
+ [string]$Result,
+ [string]$Message)
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+ 'result' = $Result
+ }
+ }
+}
<########################################
# Private functions.
@@ -123,7 +137,8 @@ function Format-LoggingCommandData {
foreach ($mapping in $script:loggingCommandEscapeMappings) {
$Value = $Value.Replace($mapping.Token, $mapping.Replacement)
}
- } else {
+ }
+ else {
for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
$mapping = $script:loggingCommandEscapeMappings[$i]
$Value = $Value.Replace($mapping.Replacement, $mapping.Token)
@@ -156,7 +171,8 @@ function Format-LoggingCommand {
if ($first) {
$null = $sb.Append(' ')
$first = $false
- } else {
+ }
+ else {
$null = $sb.Append(';')
}
@@ -193,7 +209,8 @@ function Write-LoggingCommand {
$command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
if ($AsOutput) {
$command
- } else {
+ }
+ else {
Write-Host $command
}
}
@@ -212,12 +229,12 @@ function Write-LogIssue {
[switch]$AsOutput)
$command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
- 'type' = $Type
- 'code' = $ErrCode
- 'sourcepath' = $SourcePath
- 'linenumber' = $LineNumber
- 'columnnumber' = $ColumnNumber
- }
+ 'type' = $Type
+ 'code' = $ErrCode
+ 'sourcepath' = $SourcePath
+ 'linenumber' = $LineNumber
+ 'columnnumber' = $ColumnNumber
+ }
if ($AsOutput) {
return $command
}
@@ -229,7 +246,8 @@ function Write-LogIssue {
$foregroundColor = [System.ConsoleColor]::Red
$backgroundColor = [System.ConsoleColor]::Black
}
- } else {
+ }
+ else {
$foregroundColor = $host.PrivateData.WarningForegroundColor
$backgroundColor = $host.PrivateData.WarningBackgroundColor
if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
index 6cd0a3400e..6a0b2255e9 100755
--- a/eng/common/pipeline-logging-functions.sh
+++ b/eng/common/pipeline-logging-functions.sh
@@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
local function_args=()
local message=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-category|-c)
telemetry_category=$2
@@ -48,7 +48,7 @@ function Write-PipelineTaskError {
local force=false
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-type|-t)
message_type=$2
@@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
local is_multi_job_variable=true
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-name|-n)
name=$2
@@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
local prepend_path=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-path|-p)
prepend_path=$2
@@ -179,4 +179,28 @@ function Write-PipelinePrependPath {
if [[ "$ci" == true ]]; then
echo "##vso[task.prependpath]$prepend_path"
fi
-}
\ No newline at end of file
+}
+
+function Write-PipelineSetResult {
+ local result=''
+ local message=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -result|-r)
+ result=$2
+ shift
+ ;;
+ -message|-m)
+ message=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.complete result=$result;]$message"
+ fi
+}
diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1
index 7d49744795..534f6988d5 100644
--- a/eng/common/post-build/post-build-utils.ps1
+++ b/eng/common/post-build/post-build-utils.ps1
@@ -69,9 +69,9 @@ function Trigger-Subscription([string]$SubscriptionId) {
function Validate-MaestroVars {
try {
- Get-Variable MaestroApiEndPoint -Scope Global | Out-Null
- Get-Variable MaestroApiVersion -Scope Global | Out-Null
- Get-Variable MaestroApiAccessToken -Scope Global | Out-Null
+ Get-Variable MaestroApiEndPoint | Out-Null
+ Get-Variable MaestroApiVersion | Out-Null
+ Get-Variable MaestroApiAccessToken | Out-Null
if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index 650b13b089..2427ca6b6a 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -10,21 +10,27 @@ param(
[Parameter(Mandatory=$false)][string] $EnableNugetValidation,
[Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
+ [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
)
try {
. $PSScriptRoot\post-build-utils.ps1
- # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
- $darc = Get-Darc "1.1.0-beta.20418.1"
+
+ $darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new()
if ("" -ne $ArtifactsPublishingAdditionalParameters) {
- $optionalParams.Add("artifact-publishing-parameters") | Out-Null
+ $optionalParams.Add("--artifact-publishing-parameters") | Out-Null
$optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
}
+ if ("" -ne $SymbolPublishingAdditionalParameters) {
+ $optionalParams.Add("--symbol-publishing-parameters") | Out-Null
+ $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
+ }
+
if ("false" -eq $WaitPublishingFinish) {
$optionalParams.Add("--no-wait") | Out-Null
}
@@ -54,7 +60,7 @@ try {
--id $buildId `
--publishing-infra-version $PublishingInfraVersion `
--default-channels `
- --source-branch master `
+ --source-branch main `
--azdev-pat $AzdoToken `
--bar-uri $MaestroApiEndPoint `
--password $MaestroToken `
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
index c7e7ae67d8..8c554729b6 100644
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -14,7 +14,7 @@ param(
$global:RepoFiles = @{}
# Maximum number of jobs to run in parallel
-$MaxParallelJobs = 6
+$MaxParallelJobs = 16
# Wait time between check for system load
$SecondsBetweenLoadChecks = 10
@@ -161,9 +161,12 @@ $ValidatePackage = {
function CheckJobResult(
$result,
$packagePath,
- [ref]$ValidationFailures) {
- if ($jobResult.result -ne '0') {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+ [ref]$ValidationFailures,
+ [switch]$logErrors) {
+ if ($result -ne '0') {
+ if ($logErrors) {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+ }
$ValidationFailures.Value++
}
}
@@ -228,16 +231,14 @@ function ValidateSourceLinkLinks {
foreach ($Job in @(Get-Job -State 'Completed')) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
- CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
Remove-Job -Id $Job.Id
}
}
foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
- if ($jobResult -ne '0') {
- $ValidationFailures++
- }
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
Remove-Job -Id $Job.Id
}
if ($ValidationFailures -gt 0) {
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
index fcc6019b49..788321d773 100644
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -1,30 +1,49 @@
param(
- [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
- [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
- [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
- [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
- [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols
+ [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
+ [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
+ [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
+ [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
+ [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
+ [Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols
)
# Maximum number of jobs to run in parallel
-$MaxParallelJobs = 6
+$MaxParallelJobs = 16
+
+# Max number of retries
+$MaxRetry = 5
# Wait time between check for system load
$SecondsBetweenLoadChecks = 10
+# Set error codes
+Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
+Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
+
+$WindowsPdbVerificationParam = ""
+if ($CheckForWindowsPdbs) {
+ $WindowsPdbVerificationParam = "--windows-pdbs"
+}
+
$CountMissingSymbols = {
param(
- [string] $PackagePath # Path to a NuGet package
+ [string] $PackagePath, # Path to a NuGet package
+ [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
)
. $using:PSScriptRoot\..\tools.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
+ Write-Host "Validating $PackagePath "
+
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
Write-PipelineTaskError "Input file does not exist: $PackagePath"
- return -2
+ return [pscustomobject]@{
+ result = $using:ERROR_FILEDOESNOTEXIST
+ packagePath = $PackagePath
+ }
}
# Extensions for which we'll look for symbols
@@ -45,24 +64,25 @@ $CountMissingSymbols = {
Write-Host "Something went wrong extracting $PackagePath"
Write-Host $_
return [pscustomobject]@{
- result = -1
+ result = $using:ERROR_BADEXTRACT
packagePath = $PackagePath
}
}
Get-ChildItem -Recurse $ExtractPath |
- Where-Object {$RelevantExtensions -contains $_.Extension} |
- ForEach-Object {
- $FileName = $_.FullName
- if ($FileName -Match '\\ref\\') {
- Write-Host "`t Ignoring reference assembly file " $FileName
- return
- }
+ Where-Object { $RelevantExtensions -contains $_.Extension } |
+ ForEach-Object {
+ $FileName = $_.FullName
+ if ($FileName -Match '\\ref\\') {
+ Write-Host "`t Ignoring reference assembly file " $FileName
+ return
+ }
- $FirstMatchingSymbolDescriptionOrDefault = {
+ $FirstMatchingSymbolDescriptionOrDefault = {
param(
- [string] $FullPath, # Full path to the module that has to be checked
- [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+ [string] $FullPath, # Full path to the module that has to be checked
+ [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+ [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
[string] $SymbolsPath
)
@@ -87,56 +107,67 @@ $CountMissingSymbols = {
# DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
-
+
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
- & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+ $totalRetries = 0
- if (Test-Path $PdbPath) {
- return 'PDB'
- }
- elseif (Test-Path $NGenPdb) {
- return 'NGen PDB'
- }
- elseif (Test-Path $SODbg) {
- return 'DBG for SO'
- }
- elseif (Test-Path $DylibDwarf) {
- return 'Dwarf for Dylib'
- }
- elseif (Test-Path $SymbolPath) {
- return 'Module'
- }
- else {
- return $null
+ while ($totalRetries -lt $using:MaxRetry) {
+ # Save the output and get diagnostic output
+ $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
+
+ if (Test-Path $PdbPath) {
+ return 'PDB'
+ }
+ elseif (Test-Path $NGenPdb) {
+ return 'NGen PDB'
+ }
+ elseif (Test-Path $SODbg) {
+ return 'DBG for SO'
+ }
+ elseif (Test-Path $DylibDwarf) {
+ return 'Dwarf for Dylib'
+ }
+ elseif (Test-Path $SymbolPath) {
+ return 'Module'
+ }
+ elseif ($output.Contains("503 Service Unavailable")) {
+ # If we got a 503 error, we should retry.
+ $totalRetries++
+ }
+ else {
+ return $null
+ }
}
+
+ return $null
}
- $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath
- $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath
+ $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam
+ $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam
- Write-Host -NoNewLine "`t Checking file " $FileName "... "
+ Write-Host -NoNewLine "`t Checking file " $FileName "... "
- if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
- Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
+ if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
+ Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
+ }
+ else {
+ $MissingSymbols++
+
+ if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
+ Write-Host 'No symbols found on MSDL or SymWeb!'
}
else {
- $MissingSymbols++
-
- if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
- Write-Host 'No symbols found on MSDL or SymWeb!'
+ if ($SymbolsOnMSDL -eq $null) {
+ Write-Host 'No symbols found on MSDL!'
}
else {
- if ($SymbolsOnMSDL -eq $null) {
- Write-Host 'No symbols found on MSDL!'
- }
- else {
- Write-Host 'No symbols found on SymWeb!'
- }
+ Write-Host 'No symbols found on SymWeb!'
}
}
}
+ }
if ($using:Clean) {
Remove-Item $ExtractPath -Recurse -Force
@@ -145,24 +176,31 @@ $CountMissingSymbols = {
Pop-Location
return [pscustomobject]@{
- result = $MissingSymbols
- packagePath = $PackagePath
- }
+ result = $MissingSymbols
+ packagePath = $PackagePath
+ }
}
function CheckJobResult(
- $result,
- $packagePath,
- [ref]$DupedSymbols,
- [ref]$TotalFailures) {
- if ($result -eq '-1') {
+ $result,
+ $packagePath,
+ [ref]$DupedSymbols,
+ [ref]$TotalFailures) {
+ if ($result -eq $ERROR_BADEXTRACT) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
$DupedSymbols.Value++
}
- elseif ($jobResult.result -ne '0') {
+ elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
+ $TotalFailures.Value++
+ }
+ elseif ($result -gt '0') {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
$TotalFailures.Value++
}
+ else {
+ Write-Host "All symbols verified for package $packagePath"
+ }
}
function CheckSymbolsAvailable {
@@ -192,9 +230,7 @@ function CheckSymbolsAvailable {
return
}
- Write-Host "Validating $FileName "
-
- Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null
+ Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count
diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
index e159c6f184..f55c43c6f4 100644
--- a/eng/common/sdk-task.ps1
+++ b/eng/common/sdk-task.ps1
@@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
- $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty
+ $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index b681d797cd..81b729f74a 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -87,10 +87,6 @@ try {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
- if ($UpdateBaseline) {
- & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
- }
-
if ($TsaPublish) {
if ($TsaBranchName -and $BuildNumber) {
if (-not $TsaRepositoryName) {
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index a68bf0b88e..1fe9271193 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
-try {
- # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
- Write-Host 'Downloading gdn folder from internal config repostiory...'
- Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
- if (Test-Path $gdnFolder) {
- # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
- Remove-Item -Force -Recurse $gdnFolder
- }
- [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
- Write-Host $gdnFolder
- ExitWithExitCode 0
-} catch [System.Net.WebException] { } # Catch and ignore webexception
+
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host 'Initializing Guardian...'
@@ -57,7 +46,6 @@ try {
Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
- & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
ExitWithExitCode 0
}
catch {
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index 968b39bef5..3bd8b29ebd 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1
deleted file mode 100644
index d8fd2d82a6..0000000000
--- a/eng/common/sdl/push-gdn.ps1
+++ /dev/null
@@ -1,69 +0,0 @@
-Param(
- [string] $Repository,
- [string] $BranchName='master',
- [string] $GdnFolder,
- [string] $AzureDevOpsAccessToken,
- [string] $PushReason
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-$disableConfigureToolsetImport = $true
-$LASTEXITCODE = 0
-
-try {
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- # We create the temp directory where we'll store the sdl-config repository
- $sdlDir = Join-Path $env:TEMP 'sdl'
- if (Test-Path $sdlDir) {
- Remove-Item -Force -Recurse $sdlDir
- }
-
- Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir"
- git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- # We copy the .gdn folder from our local run into the git repository so it can be committed
- $sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn'
- if (Get-Command Robocopy) {
- Robocopy /S $GdnFolder $sdlRepositoryFolder
- } else {
- rsync -r $GdnFolder $sdlRepositoryFolder
- }
- # cd to the sdl-config directory so we can run git there
- Push-Location $sdlDir
- # git add . --> git commit --> git push
- Write-Host 'git add .'
- git add .
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
- git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- Write-Host 'git push'
- git push
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
-
- # Return to the original directory
- Pop-Location
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index c64c4f5686..4a32181fd8 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -45,6 +45,7 @@ jobs:
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ checkDownloadedFiles: true
- ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
@@ -57,6 +58,7 @@ jobs:
downloadType: specific files
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ checkDownloadedFiles: true
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@@ -83,7 +85,7 @@ jobs:
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
+ -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index e78ed9a1c6..8669679348 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -24,9 +24,9 @@ parameters:
enablePublishBuildAssets: false
enablePublishTestResults: false
enablePublishUsingPipelines: false
- useBuildManifest: false
mergeTestResults: false
testRunTitle: ''
+ testResultsFormat: ''
name: ''
preSteps: []
runAsPublic: false
@@ -131,8 +131,8 @@ jobs:
- task: RichCodeNavIndexer@0
displayName: RichCodeNav Upload
inputs:
- languages: 'csharp'
- environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }}
+ languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+ environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
continueOnError: true
@@ -202,9 +202,9 @@ jobs:
continueOnError: true
condition: always()
- - ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
- task: PublishTestResults@2
- displayName: Publish Test Results
+ displayName: Publish XUnit Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
@@ -213,6 +213,17 @@ jobs:
mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true
condition: always()
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish TRX Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
- ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
@@ -231,12 +242,3 @@ jobs:
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
-
- - ${{ if eq(parameters.useBuildManifest, true) }}:
- - task: PublishBuildArtifacts@1
- displayName: Publish Build Manifest
- inputs:
- PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
- PublishLocation: Container
- ArtifactName: BuildManifests
- continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml
new file mode 100644
index 0000000000..b27d6faf30
--- /dev/null
+++ b/eng/common/templates/job/onelocbuild.yml
@@ -0,0 +1,83 @@
+parameters:
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool:
+ vmImage: vs2017-win2016
+
+ CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+ GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+ SourcesDirectory: $(Build.SourcesDirectory)
+ CreatePr: true
+ AutoCompletePr: false
+ UseCheckedInLocProjectJson: false
+ LanguageSet: VS_Main_Languages
+ LclSource: lclFilesInRepo
+ LclPackageId: ''
+ RepoType: gitHub
+ condition: ''
+
+jobs:
+- job: OneLocBuild
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: OneLocBuild
+
+ pool: ${{ parameters.pool }}
+
+ variables:
+ - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+ - name: _GenerateLocProjectArguments
+ value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+ -LanguageSet "${{ parameters.LanguageSet }}"
+ -CreateNeutralXlfs
+ - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+ - name: _GenerateLocProjectArguments
+ value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+
+
+ steps:
+ - task: Powershell@2
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+ arguments: $(_GenerateLocProjectArguments)
+ displayName: Generate LocProject.json
+ condition: ${{ parameters.condition }}
+
+ - task: OneLocBuild@2
+ displayName: OneLocBuild
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ inputs:
+ locProj: Localize/LocProject.json
+ outDir: $(Build.ArtifactStagingDirectory)
+ lclSource: ${{ parameters.LclSource }}
+ lclPackageId: ${{ parameters.LclPackageId }}
+ isCreatePrSelected: ${{ parameters.CreatePr }}
+ ${{ if eq(parameters.CreatePr, true) }}:
+ isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+ packageSourceAuth: patAuth
+ patVariable: ${{ parameters.CeapexPat }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ repoType: ${{ parameters.RepoType }}
+ gitHubPatVariable: "${{ parameters.GithubPat }}"
+ condition: ${{ parameters.condition }}
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish Localization Files
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
+
+ - task: PublishBuildArtifacts@1
+ displayName: Publish LocProject.json
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/Localize/'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
\ No newline at end of file
diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml
deleted file mode 100644
index f877fd7a89..0000000000
--- a/eng/common/templates/job/performance.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-parameters:
- steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo)
- variables: [] # optional -- list of additional variables to send to the template
- jobName: '' # required -- job name
- displayName: '' # optional -- display name for the job. Will use jobName if not passed
- pool: '' # required -- name of the Build pool
- container: '' # required -- name of the container
- osGroup: '' # required -- operating system for the job
- extraSetupParameters: '' # optional -- extra arguments to pass to the setup script
- frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against
- continueOnError: 'false' # optional -- determines whether to continue the build if the step errors
- dependsOn: '' # optional -- dependencies of the job
- timeoutInMinutes: 320 # optional -- timeout for the job
- enableTelemetry: false # optional -- enable for telemetry
-
-jobs:
-- template: ../jobs/jobs.yml
- parameters:
- dependsOn: ${{ parameters.dependsOn }}
- enableTelemetry: ${{ parameters.enableTelemetry }}
- enablePublishBuildArtifacts: true
- continueOnError: ${{ parameters.continueOnError }}
-
- jobs:
- - job: '${{ parameters.jobName }}'
-
- ${{ if ne(parameters.displayName, '') }}:
- displayName: '${{ parameters.displayName }}'
- ${{ if eq(parameters.displayName, '') }}:
- displayName: '${{ parameters.jobName }}'
-
- timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-
- variables:
-
- - ${{ each variable in parameters.variables }}:
- - ${{ if ne(variable.name, '') }}:
- - name: ${{ variable.name }}
- value: ${{ variable.value }}
- - ${{ if ne(variable.group, '') }}:
- - group: ${{ variable.group }}
-
- - IsInternal: ''
- - HelixApiAccessToken: ''
- - HelixPreCommand: ''
-
- - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - ${{ if eq( parameters.osGroup, 'Windows_NT') }}:
- - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
- - IsInternal: -Internal
- - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- - IsInternal: --internal
-
- - group: DotNet-HelixApi-Access
- - group: dotnet-benchview
-
- workspace:
- clean: all
- pool:
- ${{ parameters.pool }}
- container: ${{ parameters.container }}
- strategy:
- matrix:
- ${{ each framework in parameters.frameworks }}:
- ${{ framework }}:
- _Framework: ${{ framework }}
- steps:
- - checkout: self
- clean: true
- # Run all of the steps to setup repo
- - ${{ each step in parameters.steps }}:
- - ${{ step }}
- - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }}
- displayName: Performance Setup (Windows)
- condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
- continueOnError: ${{ parameters.continueOnError }}
- - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }}
- displayName: Performance Setup (Unix)
- condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
- continueOnError: ${{ parameters.continueOnError }}
- - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments)
- displayName: Run ci setup script
- # Run perf testing in helix
- - template: /eng/common/templates/steps/perf-send-to-helix.yml
- parameters:
- HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
- HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)'
- HelixAccessToken: $(HelixApiAccessToken)
- HelixTargetQueues: $(Queue)
- HelixPreCommands: $(HelixPreCommand)
- Creator: $(Creator)
- WorkItemTimeout: 4:00 # 4 hours
- WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy
- CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions
\ No newline at end of file
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index d0c3cc2b3b..3b9e2524ff 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -37,6 +37,7 @@ jobs:
- name: _BuildConfig
value: ${{ parameters.configuration }}
- group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
@@ -51,12 +52,19 @@ jobs:
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0
+ - task: PowerShell@2
+ displayName: Enable cross-org NuGet feed authentication
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
+
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
new file mode 100644
index 0000000000..aad4146492
--- /dev/null
+++ b/eng/common/templates/job/source-build.yml
@@ -0,0 +1,57 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. The template produces a server job with a
+ # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+ # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+ jobNamePrefix: 'Source_Build'
+
+ # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+ # managed-only repositories. This is an object with these properties:
+ #
+ # name: ''
+ # The name of the job. This is included in the job ID.
+ # targetRID: ''
+ # The name of the target RID to use, instead of the one auto-detected by Arcade.
+ # nonPortable: false
+ # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+ # linux-x64), and compiling against distro-provided packages rather than portable ones.
+ # container: ''
+ # A container to use. Runs in docker.
+ # pool: {}
+ # A pool to use. Runs directly on an agent.
+ # buildScript: ''
+ # Specifies the build script to invoke to perform the build in the repo. The default
+ # './build.sh' should work for typical Arcade repositories, but this is customizable for
+ # difficult situations.
+ # jobProperties: {}
+ # A list of job properties to inject at the top level, for potential extensibility beyond
+ # container and pool.
+ platform: {}
+
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ defaultContainerHostPool:
+ vmImage: ubuntu-20.04
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+ displayName: Source-Build (${{ parameters.platform.name }})
+
+ ${{ each property in parameters.platform.jobProperties }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ ${{ if ne(parameters.platform.container, '') }}:
+ container: ${{ parameters.platform.container }}
+
+ ${{ if eq(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.defaultContainerHostPool }}
+ ${{ if ne(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.platform.pool }}
+
+ workspace:
+ clean: all
+
+ steps:
+ - template: /eng/common/templates/steps/source-build.yml
+ parameters:
+ platform: ${{ parameters.platform }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
new file mode 100644
index 0000000000..a649d2b599
--- /dev/null
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -0,0 +1,58 @@
+parameters:
+ runAsPublic: false
+ sourceIndexPackageVersion: 1.0.1-20210421.1
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+ preSteps: []
+ binlogPath: artifacts/log/Debug/Build.binlog
+ pool:
+ vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+ variables:
+ - name: SourceIndexPackageVersion
+ value: ${{ parameters.sourceIndexPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
+ - name: BinlogPath
+ value: ${{ parameters.binlogPath }}
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: source-dot-net stage1 variables
+
+ pool: ${{ parameters.pool }}
+ steps:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk 3.1
+ inputs:
+ packageType: sdk
+ version: 3.1.x
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk
+ inputs:
+ useGlobalJson: true
+
+ - script: |
+ dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+ displayName: Download Tools
+
+ - script: ${{ parameters.sourceIndexBuildCommand }}
+ displayName: Build Repository
+
+ - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+ env:
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+ displayName: Upload stage1 artifacts to source index
+ env:
+ BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index c08225a9a9..a1f8fce96c 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -7,7 +7,14 @@ parameters:
# Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false
-
+
+ # Optional: Enable running the source-build jobs to build repo from source
+ enableSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
graphFileGeneration:
# Optional: Enable generating the graph files at the end of the build
enabled: false
@@ -24,6 +31,9 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
+ enableSourceIndex: false
+ sourceIndexParams: {}
+
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@@ -43,7 +53,22 @@ jobs:
name: ${{ job.job }}
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - template: /eng/common/templates/jobs/source-build.yml
+ parameters:
+ allCompletedJobId: Source_Build_Complete
+ ${{ each parameter in parameters.sourceBuildParameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+ - template: ../job/source-index-stage1.yml
+ parameters:
+ runAsPublic: ${{ parameters.runAsPublic }}
+ ${{ each parameter in parameters.sourceIndexParams }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
@@ -55,6 +80,8 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - Source_Build_Complete
pool:
vmImage: vs2017-win2016
runAsPublic: ${{ parameters.runAsPublic }}
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
new file mode 100644
index 0000000000..00aa98eb3b
--- /dev/null
+++ b/eng/common/templates/jobs/source-build.yml
@@ -0,0 +1,46 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+ # well as an optional server job that completes when all platform jobs complete.
+
+ # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+ # not included. Existing repo pipelines can use this job depend on all source-build jobs
+ # completing without maintaining a separate list of every single job ID: just depend on this one
+ # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+ allCompletedJobId: ''
+
+ # See /eng/common/templates/job/source-build.yml
+ jobNamePrefix: 'Source_Build'
+
+ # This is the default platform provided by Arcade, intended for use by a managed-only repo.
+ defaultManagedPlatform:
+ name: 'Managed'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+ # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+ # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+ # one job runs on 'defaultManagedPlatform'.
+ platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+ - job: ${{ parameters.allCompletedJobId }}
+ displayName: Source-Build Complete
+ pool: server
+ dependsOn:
+ - ${{ each platform in parameters.platforms }}:
+ - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+ - ${{ if eq(length(parameters.platforms), 0) }}:
+ - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ platform }}
+
+- ${{ if eq(length(parameters.platforms), 0) }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ parameters.defaultManagedPlatform }}
diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml
index a0a8074282..4e51e472e2 100644
--- a/eng/common/templates/phases/publish-build-assets.yml
+++ b/eng/common/templates/phases/publish-build-assets.yml
@@ -20,6 +20,7 @@ phases:
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
index 7ae5255921..58fa9a35b8 100644
--- a/eng/common/templates/post-build/channels/generic-internal-channel.yml
+++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml
@@ -58,6 +58,7 @@ stages:
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -124,6 +125,7 @@ stages:
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml
index 6cf39dbb29..b50c0b3bdb 100644
--- a/eng/common/templates/post-build/channels/generic-public-channel.yml
+++ b/eng/common/templates/post-build/channels/generic-public-channel.yml
@@ -56,6 +56,7 @@ stages:
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -123,6 +124,7 @@ stages:
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
+ checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index df06f5371e..4f79cf0f33 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -32,7 +32,6 @@ parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
signingValidationAdditionalParameters: ''
- useBuildManifest: false
# Which stages should finish execution before post-build stages start
validateDependsOn:
@@ -54,9 +53,6 @@ parameters:
NETCoreExperimentalChannelId: 562
NetEngServicesIntChannelId: 678
NetEngServicesProdChannelId: 679
- Net5Preview8ChannelId: 1155
- Net5RC1ChannelId: 1157
- Net5RC2ChannelId: 1329
NetCoreSDK313xxChannelId: 759
NetCoreSDK313xxInternalChannelId: 760
NetCoreSDK314xxChannelId: 921
@@ -65,177 +61,180 @@ parameters:
VS167ChannelId: 1011
VS168ChannelId: 1154
VSMasterChannelId: 1012
-
-stages:
-- stage: Validate
- dependsOn: ${{ parameters.validateDependsOn }}
- displayName: Validate Build Assets
- variables:
- - template: common-variables.yml
- jobs:
- - template: setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ VS169ChannelId: 1473
+ VS1610ChannelId: 1692
- - job:
- displayName: Post-build Checks
- dependsOn: setupMaestroVars
- variables:
- - name: TargetChannels
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - task: PowerShell@2
- displayName: Maestro Channels Consistency
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
- arguments: -PromoteToChannels "$(TargetChannels)"
- -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
-
- - job:
- displayName: NuGet Validation
- dependsOn: setupMaestroVars
- condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
- pool:
- vmImage: 'windows-2019'
- variables:
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- steps:
- - task: DownloadBuildArtifacts@0
- displayName: Download Package Artifacts
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- artifactName: PackageArtifacts
-
- - task: PowerShell@2
- displayName: Validate
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
- arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
-
- - job:
- displayName: Signing Validation
- dependsOn: setupMaestroVars
- condition: eq( ${{ parameters.enableSigningValidation }}, 'true')
+stages:
+- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ - stage: Validate
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Validate Build Assets
variables:
- template: common-variables.yml
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - ${{ if eq(parameters.useBuildManifest, true) }}:
+ jobs:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+ - job:
+ displayName: Post-build Checks
+ dependsOn: setupMaestroVars
+ variables:
+ - name: TargetChannels
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: PowerShell@2
+ displayName: Maestro Channels Consistency
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+ arguments: -PromoteToChannels "$(TargetChannels)"
+ -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
+
+ - job:
+ displayName: NuGet Validation
+ dependsOn: setupMaestroVars
+ condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
+ pool:
+ vmImage: 'windows-2019'
+ variables:
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ steps:
- task: DownloadBuildArtifacts@0
- displayName: Download build manifest
+ displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
- artifactName: BuildManifests
- - task: DownloadBuildArtifacts@0
- displayName: Download Package Artifacts
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- artifactName: PackageArtifacts
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
- # otherwise it'll complain about accessing a private feed.
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: PowerShell@2
- displayName: Enable cross-org publishing
- inputs:
- filePath: eng\common\enable-cross-org-publishing.ps1
- arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
-
- # Signing validation will optionally work with the buildmanifest file which is downloaded from
- # Azure DevOps above.
- - task: PowerShell@2
- displayName: Validate
- inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task SigningValidation -restore -msbuildEngine vs
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
- /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
- ${{ parameters.signingValidationAdditionalParameters }}
-
- - template: ../steps/publish-logs.yml
- parameters:
- StageLabel: 'Validation'
- JobLabel: 'Signing'
-
- - job:
- displayName: SourceLink Validation
- dependsOn: setupMaestroVars
- condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
- variables:
- - template: common-variables.yml
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - task: DownloadBuildArtifacts@0
- displayName: Download Blob Artifacts
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- artifactName: BlobArtifacts
-
- - task: PowerShell@2
- displayName: Validate
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
- arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -ExtractPath $(Agent.BuildDirectory)/Extract/
- -GHRepoName $(Build.Repository.Name)
- -GHCommit $(Build.SourceVersion)
- -SourcelinkCliVersion $(SourceLinkCLIVersion)
- continueOnError: true
-
- - template: /eng/common/templates/job/execute-sdl.yml
- parameters:
- enable: ${{ parameters.SDLValidationParameters.enable }}
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+ arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
+ -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
+
+ - job:
+ displayName: Signing Validation
dependsOn: setupMaestroVars
- additionalParameters: ${{ parameters.SDLValidationParameters.params }}
- continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
- artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
- downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
+ condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
+ variables:
+ - template: common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ # Signing validation will optionally work with the buildmanifest file which is downloaded from
+ # Azure DevOps above.
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task SigningValidation -restore -msbuildEngine vs
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+ ${{ parameters.signingValidationAdditionalParameters }}
+
+ - template: ../steps/publish-logs.yml
+ parameters:
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
+
+ - job:
+ displayName: SourceLink Validation
+ dependsOn: setupMaestroVars
+ condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
+ variables:
+ - template: common-variables.yml
+ - name: AzDOProjectName
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+ - name: AzDOPipelineId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+ - name: AzDOBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: BlobArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+ -ExtractPath $(Agent.BuildDirectory)/Extract/
+ -GHRepoName $(Build.Repository.Name)
+ -GHCommit $(Build.SourceVersion)
+ -SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
+
+ - template: /eng/common/templates/job/execute-sdl.yml
+ parameters:
+ enable: ${{ parameters.SDLValidationParameters.enable }}
+ dependsOn: setupMaestroVars
+ additionalParameters: ${{ parameters.SDLValidationParameters.params }}
+ continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+ artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
+ downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- stage: publish_using_darc
- dependsOn: Validate
+ ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.publishDependsOn }}
+ ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
variables:
- template: common-variables.yml
@@ -248,6 +247,7 @@ stages:
- job:
displayName: Publish Using Darc
dependsOn: setupMaestroVars
+ timeoutInMinutes: 120
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
@@ -264,6 +264,8 @@ stages:
-MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
-PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
+ -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+ -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
@@ -298,54 +300,6 @@ stages:
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
- - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net5_Preview8_Publish'
- channelName: '.NET 5 Preview 8'
- akaMSChannelName: 'net5/preview8'
- channelId: ${{ parameters.Net5Preview8ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net5_RC1_Publish'
- channelName: '.NET 5 RC 1'
- akaMSChannelName: 'net5/rc1'
- channelId: ${{ parameters.Net5RC1ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net5_RC2_Publish'
- channelName: '.NET 5 RC 2'
- akaMSChannelName: 'net5/rc2'
- channelId: ${{ parameters.Net5RC2ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
-
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
@@ -603,3 +557,33 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_9_Publishing'
+ channelName: 'VS 16.9'
+ channelId: ${{ parameters.VS169ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_10_Publishing'
+ channelName: 'VS 16.10'
+ channelId: ${{ parameters.VS1610ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml
index d0cbfb6c6f..4a22b2e6f6 100644
--- a/eng/common/templates/post-build/setup-maestro-vars.yml
+++ b/eng/common/templates/post-build/setup-maestro-vars.yml
@@ -18,6 +18,7 @@ jobs:
inputs:
buildType: current
artifactName: ReleaseConfigs
+ checkDownloadedFiles: true
- task: PowerShell@2
name: setReleaseVars
diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml
deleted file mode 100644
index e003fe2ef2..0000000000
--- a/eng/common/templates/steps/perf-send-to-helix.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# Please remember to update the documentation if you make changes to these parameters!
-parameters:
- ProjectFile: '' # required -- project file that specifies the helix workitems
- HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
- HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
- HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
- HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
- HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
- HelixPreCommands: '' # optional -- commands to run before Helix work item execution
- HelixPostCommands: '' # optional -- commands to run after Helix work item execution
- WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
- CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
- IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
- DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
- DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
- EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
- WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
- Creator: '' # optional -- if the build is external, use this to specify who is sending the job
- DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO
- condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
- continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
-
-
-steps:
- - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog
- displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
- env:
- BuildConfig: $(_BuildConfig)
- HelixSource: ${{ parameters.HelixSource }}
- HelixType: ${{ parameters.HelixType }}
- HelixBuild: ${{ parameters.HelixBuild }}
- HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
- HelixAccessToken: ${{ parameters.HelixAccessToken }}
- HelixPreCommands: ${{ parameters.HelixPreCommands }}
- HelixPostCommands: ${{ parameters.HelixPostCommands }}
- WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
- CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
- IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
- DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
- DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
- EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
- WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
- Creator: ${{ parameters.Creator }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
- continueOnError: ${{ parameters.continueOnError }}
- - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
- displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
- env:
- BuildConfig: $(_BuildConfig)
- HelixSource: ${{ parameters.HelixSource }}
- HelixType: ${{ parameters.HelixType }}
- HelixBuild: ${{ parameters.HelixBuild }}
- HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
- HelixAccessToken: ${{ parameters.HelixAccessToken }}
- HelixPreCommands: ${{ parameters.HelixPreCommands }}
- HelixPostCommands: ${{ parameters.HelixPostCommands }}
- WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
- CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
- IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
- DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
- DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
- EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
- WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
- Creator: ${{ parameters.Creator }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
- continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
index 5eceb48725..cd02ae1607 100644
--- a/eng/common/templates/steps/send-to-helix.yml
+++ b/eng/common/templates/steps/send-to-helix.yml
@@ -18,8 +18,8 @@ parameters:
XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
- DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
- DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
+ DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+ DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml
new file mode 100644
index 0000000000..65ee5992bf
--- /dev/null
+++ b/eng/common/templates/steps/source-build.yml
@@ -0,0 +1,66 @@
+parameters:
+ # This template adds arcade-powered source-build to CI.
+
+ # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+ # infra has a careful build methodology that must be followed. For example, a repo
+ # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+ # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+ # GitHub. Using this steps template leaves room for that infra to be included.
+
+ # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+ # for details. The entire object is described in the 'job' template for simplicity, even though
+ # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+ platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+ set -x
+ df -h
+
+ buildConfig=Release
+ # Check if AzDO substitutes in a build config from a variable, and use it if so.
+ if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+ buildConfig='$(_BuildConfig)'
+ fi
+
+ officialBuildArgs=
+ if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+ officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+ fi
+
+ targetRidArgs=
+ if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+ targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+ fi
+
+ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+ --configuration $buildConfig \
+ --restore --build --pack --publish -bl \
+ $officialBuildArgs \
+ $targetRidArgs \
+ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+ /p:ArcadeBuildFromSource=true
+ displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+ displayName: Prepare BuildLogs staging directory
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)'
+ Contents: |
+ **/*.log
+ **/*.binlog
+ artifacts/source-build/self/prebuilt-report/**
+ TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+ CleanTargetFolder: true
+ continueOnError: true
+ condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+ displayName: Publish BuildLogs
+ inputs:
+ targetPath: '$(Build.StagingDirectory)/BuildLogs'
+ artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+ continueOnError: true
+ condition: succeededOrFailed()
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 37a3a3d680..d52467eea1 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -57,15 +57,11 @@ set-strictmode -version 2.0
$ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
-# If specified, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
+# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
-# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors).
-# otherwise will fetch from public location.
-[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false }
-
function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
}
@@ -145,7 +141,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
# otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
- if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+ if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR
} else {
$dotnetRoot = Join-Path $RepoRoot '.dotnet'
@@ -173,7 +169,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
try {
- Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt'
+ Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
} catch {
# Somebody beat us
Remove-Item -Path $sdkCacheFileTemp
@@ -197,46 +193,37 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
if (!(Test-Path $installScript)) {
- create-directory $dotnetroot
-
- if ($useDefaultDotnetInstall)
- {
- $progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit
+ Create-Directory $dotnetRoot
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- $maxretries = 5
- $retries = 1
+ $maxRetries = 5
+ $retries = 1
- $uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1"
+ $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
- while($true) {
- try {
- write-host "get $uri"
- invoke-webrequest $uri -outfile $installscript
- break
- }
- catch {
- write-host "failed to download '$uri'"
- write-error $_.exception.message -erroraction continue
- }
+ while($true) {
+ try {
+ Write-Host "GET $uri"
+ Invoke-WebRequest $uri -OutFile $installScript
+ break
+ }
+ catch {
+ Write-Host "Failed to download '$uri'"
+ Write-Error $_.Exception.Message -ErrorAction Continue
+ }
- if (++$retries -le $maxretries) {
- $delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff
- write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)."
- start-sleep -seconds $delayinseconds
- }
- else {
- throw "unable to download file in $maxretries attempts."
- }
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
}
- }
- else
- {
- # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path.
- # See https://github.com/dotnet/arcade/issues/6047 for details
- $engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1')
- Copy-Item $engCommonCopy -Destination $installScript -Force
+ else {
+ throw "Unable to download file in $maxRetries attempts."
+ }
+
}
}
+
return $installScript
}
@@ -312,9 +299,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
return $global:_MSBuildExe
}
- $vsMinVersionReqdStr = '16.5'
+ # Minimum VS version to require.
+ $vsMinVersionReqdStr = '16.8'
$vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
+ # If the version of msbuild is going to be xcopied,
+ # use this version. Version matches a package here:
+ # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
+ $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
$vsMinVersion = [Version]::new($vsMinVersionStr)
@@ -349,23 +342,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
$xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
} else {
- #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download
+ #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
if($vsMinVersion -lt $vsMinVersionReqd){
- Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible"
- $vsMajorVersion = $vsMinVersionReqd.Major
- $vsMinorVersion = $vsMinVersionReqd.Minor
+ Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+ $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
}
else{
+ # If the VS version IS compatible, look for an xcopy msbuild package
+ # with a version matching VS.
+ # Note: If this version does not exist, then an explicit version of xcopy msbuild
+ # can be specified in global.json. This will be required for pre-release versions of msbuild.
$vsMajorVersion = $vsMinVersion.Major
$vsMinorVersion = $vsMinVersion.Minor
+ $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
}
-
- $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha"
}
$vsInstallDir = $null
if ($xcopyMSBuildVersion.Trim() -ine "none") {
$vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+ if ($vsInstallDir -eq $null) {
+ throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+ }
}
if ($vsInstallDir -eq $null) {
throw 'Unable to find Visual Studio that has required version and components installed'
@@ -441,11 +439,26 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
Write-Host 'Downloading vswhere'
- try {
- Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
- }
- catch {
- Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ $maxRetries = 5
+ $retries = 1
+
+ while($true) {
+ try {
+ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+ break
+ }
+ catch{
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ }
+
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+ }
}
}
@@ -495,7 +508,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -529,13 +542,15 @@ function GetDefaultMSBuildEngine() {
function GetNuGetPackageCachePath() {
if ($env:NUGET_PACKAGES -eq $null) {
- # Use local cache on CI to ensure deterministic build,
+ # Use local cache on CI to ensure deterministic build.
+ # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
# use global cache in dev builds to avoid cost of downloading packages.
# For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
if ($useGlobalNuGetCache) {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
} else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+ $env:RESTORENOCACHE = $true
}
}
@@ -629,9 +644,26 @@ function MSBuild() {
}
$toolsetBuildProject = InitializeToolset
- $path = Split-Path -parent $toolsetBuildProject
- $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
- $args += "/logger:$path"
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
+ if (-not $selectedPath) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ ExitWithExitCode 1
+ }
+ $args += "/logger:$selectedPath"
}
MSBuild-Core @args
@@ -677,14 +709,23 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.'
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args
- if ($buildLog -ne $null) {
+ if ($null -ne $buildLog) {
Write-Host "See log: $buildLog" -ForegroundColor DarkGray
}
- ExitWithExitCode $exitCode
+ if ($ci) {
+ Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ } else {
+ ExitWithExitCode $exitCode
+ }
}
}
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index c722a05853..5fad1846e5 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -68,10 +68,6 @@ fi
runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''}
-# Determines if dotnet-install.sh comes from the eng/common folder or the internet
-# (default = public version)
-use_default_dotnet_install=${use_default_dotnet_install:-false}
-
# Resolve any symlinks in the given path.
function ResolvePath {
local path=$1
@@ -93,16 +89,16 @@ function ResolvePath {
function ReadGlobalVersion {
local key=$1
- local line=$(awk "/$key/ {print; exit}" "$global_json_file")
- local pattern="\"$key\" *: *\"(.*)\""
+ if command -v jq &> /dev/null; then
+ _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+ elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+ _ReadGlobalVersion=${BASH_REMATCH[1]}
+ fi
- if [[ ! $line =~ $pattern ]]; then
+ if [[ -z "$_ReadGlobalVersion" ]]; then
Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1
fi
-
- # return value
- _ReadGlobalVersion=${BASH_REMATCH[1]}
}
function InitializeDotNetCli {
@@ -253,7 +249,7 @@ function with_retries {
return 0
fi
- timeout=$((2**$retries-1))
+ timeout=$((3**$retries-1))
echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
sleep $timeout
done
@@ -271,30 +267,31 @@ function GetDotNetInstallScript {
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
- if [[ "$use_default_dotnet_install" == true ]]; then
- echo "Downloading '$install_script_url'"
-
- # Use curl if available, otherwise use wget
- if command -v curl > /dev/null; then
- with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
- local exit_code=$?
- Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
- ExitWithExitCode $exit_code
- }
- else
- with_retries wget -v -O "$install_script" "$install_script_url" || {
+ echo "Downloading '$install_script_url'"
+
+ # Use curl if available, otherwise use wget
+ if command -v curl > /dev/null; then
+ # first, try directly, if this fails we will retry with verbose logging
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ if command -v openssl &> /dev/null; then
+ echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+ echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ fi
+ echo "Will now retry the same URL with verbose logging."
+ with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
- fi
+ }
else
- # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path.
- # See https://github.com/dotnet/arcade/issues/6047 for details
- cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script
+ with_retries wget -v -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
fi
fi
-
# return value
_GetDotNetInstallScript="$install_script"
}
@@ -309,15 +306,17 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
- _InitializeBuildToolFramework="netcoreapp2.1"
+ _InitializeBuildToolFramework="netcoreapp3.1"
}
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
function GetNuGetPackageCachePath {
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ "$use_global_nuget_cache" == true ]]; then
export NUGET_PACKAGES="$HOME/.nuget/packages"
else
export NUGET_PACKAGES="$repo_root/.packages"
+ export RESTORENOCACHE=true
fi
fi
@@ -414,8 +413,24 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
- local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
- args=( "${args[@]}" "-logger:$logger_path" )
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
+ if [[ -z "$selectedPath" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ ExitWithExitCode 1
+ fi
+ args+=( "-logger:$selectedPath" )
fi
MSBuild-Core ${args[@]}
@@ -446,8 +461,17 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$@" || {
local exit_code=$?
- Write-PipelineTaskError "Build failed (exit code '$exit_code')."
- ExitWithExitCode $exit_code
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ echo "Build failed with exit code $exit_code. Check errors above."
+ if [[ "$ci" == "true" ]]; then
+ Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ else
+ ExitWithExitCode $exit_code
+ fi
}
}
@@ -470,8 +494,11 @@ temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json"
# determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false
-dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true
-if [[ -n "$dotnetlocal_key" ]]; then
+if command -v jq &> /dev/null; then
+ if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+ global_json_has_runtimes=true
+ fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
global_json_has_runtimes=true
fi
diff --git a/global.json b/global.json
index ee6291512f..1923fa1b2a 100644
--- a/global.json
+++ b/global.json
@@ -1,16 +1,22 @@
{
- "tools": {
- "dotnet": "3.1.102",
- "runtimes": {
- "dotnet/x64": ["$(MicrosoftNETCorePlatformsVersion)", "$(MicrosoftNETCore3PlatformsVersion)"],
- "dotnet/x86": ["$(MicrosoftNETCorePlatformsVersion)", "$(MicrosoftNETCore3PlatformsVersion)"]
- }
- },
- "msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20461.7",
- "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20461.7",
- "Microsoft.Build.Traversal": "2.1.1",
- "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
- "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
- }
+ "tools": {
+ "dotnet": "6.0.100-preview.3.21202.5",
+ "runtimes": {
+ "dotnet/x64": [
+ "$(MicrosoftNETCorePlatformsVersion)",
+ "$(MicrosoftNETCore3PlatformsVersion)"
+ ],
+ "dotnet/x86": [
+ "$(MicrosoftNETCorePlatformsVersion)",
+ "$(MicrosoftNETCore3PlatformsVersion)"
+ ]
+ }
+ },
+ "msbuild-sdks": {
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21253.2",
+ "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21253.2",
+ "Microsoft.Build.Traversal": "2.1.1",
+ "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
+ "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
}
+}
diff --git a/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs b/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs
index 2893956bef..bcac9a8bcd 100644
--- a/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs
+++ b/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs
@@ -15,14 +15,12 @@ namespace Microsoft.ML.PerformanceTests
{
public class RecommendedConfig : ManualConfig
{
- protected static readonly IReadOnlyList msbuildArguments = new List() { new MsBuildArgument($"/p:Configuration={GetBuildConfigurationName()}") };
-
public RecommendedConfig()
{
Add(DefaultConfig.Instance); // this config contains all of the basic settings (exporters, columns etc)
Add(GetJobDefinition() // job defines how many times given benchmark should be executed
- .With(msbuildArguments)
+ .WithCustomBuildConfiguration(GetBuildConfigurationName())
.With(CreateToolchain())); // toolchain is responsible for generating, building and running dedicated executable per benchmark
Add(new ExtraMetricColumn()); // an extra column that can display additional metric reported by the benchmarks
@@ -60,7 +58,7 @@ private IToolchain CreateToolchain()
csProj.Executor);
}
- private static string GetBuildConfigurationName()
+ protected static string GetBuildConfigurationName()
{
#if NETCOREAPP3_1
return "Release-netcoreapp3_1";
@@ -76,7 +74,7 @@ public class TrainConfig : RecommendedConfig
{
protected override Job GetJobDefinition()
=> Job.Dry // the "Dry" job runs the benchmark exactly once, without any warmup to mimic real-world scenario
- .With(msbuildArguments)
+ .WithCustomBuildConfiguration(GetBuildConfigurationName())
.WithLaunchCount(3); // BDN will run 3 dedicated processes, sequentially
}
}