From 3bae7d9580c3138a9bc67b49666b9a6927f176e3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 20 Apr 2020 20:39:27 -0600 Subject: [PATCH] Add script to install the .NET Core SDK This works on local dev boxes and on Azure Pipeline agents so that the SDK version we need is always installed on-demand. --- CONTRIBUTING.md | 39 ++++ README.md | 46 +++-- azure-pipelines.yml | 3 +- azure-pipelines/Get-NuGetTool.ps1 | 22 +++ azure-pipelines/Get-ProcDump.ps1 | 14 ++ azure-pipelines/Get-TempToolsPath.ps1 | 13 ++ azure-pipelines/Set-EnvVars.ps1 | 79 ++++++++ .../variables/DotNetSdkVersion.ps1 | 2 + build.sh | 175 ------------------ init.cmd | 4 + init.ps1 | 96 +++++++--- tools/Get-CatalogBin.ps1 | 31 ++++ tools/Install-DotNetSdk.ps1 | 159 ++++++++++++++++ tools/Install-NuGetCredProvider.ps1 | 74 ++++++++ 14 files changed, 533 insertions(+), 224 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 azure-pipelines/Get-NuGetTool.ps1 create mode 100644 azure-pipelines/Get-ProcDump.ps1 create mode 100644 azure-pipelines/Get-TempToolsPath.ps1 create mode 100644 azure-pipelines/Set-EnvVars.ps1 create mode 100644 azure-pipelines/variables/DotNetSdkVersion.ps1 delete mode 100755 build.sh create mode 100644 init.cmd mode change 100644 => 100755 init.ps1 create mode 100644 tools/Get-CatalogBin.ps1 create mode 100644 tools/Install-DotNetSdk.ps1 create mode 100644 tools/Install-NuGetCredProvider.ps1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..1fa778a9c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +# Contributing + +This project has adopted the [Microsoft Open Source Code of +Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct +FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) +with any additional questions or comments. + +For our general contributing guidelines please see [our dotnet/runtime contributing guide](https://github.com/dotnet/runtime/blob/master/CONTRIBUTING.md). + +## Best practices + +* Use Windows PowerShell or [PowerShell Core][pwsh] (including on Linux/OSX) to run .ps1 scripts. + Some scripts set environment variables to help you, but they are only retained if you use PowerShell as your shell. + +## Prerequisites + +All dependencies can be installed by running the `init.ps1` script at the root of the repository +using Windows PowerShell or [PowerShell Core][pwsh] (on any OS). + +The only prerequisite for building, testing, and deploying from this repository +is the [.NET SDK](https://get.dot.net/). +You should install the version specified in `global.json` or a later version within +the same major.minor.Bxx "hundreds" band. +For example if 2.2.300 is specified, you may install 2.2.300, 2.2.301, or 2.2.310 +while the 2.2.400 version would not be considered compatible by .NET SDK. +See [.NET Core Versioning](https://docs.microsoft.com/en-us/dotnet/core/versions/) for more information. + +The development experience is best with [Visual Studio][VisualStudio]. + +## Building + +This repository can be built on Windows, Linux, and OSX. + +Building, testing, and packing this repository can be done by using the standard dotnet CLI commands (e.g. `dotnet build`, `dotnet test`, `dotnet pack`, etc.). + +[pwsh]: https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-6 +[VisualStudio]: https://docs.microsoft.com/dotnet/core/install/sdk?pivots=os-windows#install-with-visual-studio diff --git a/README.md b/README.md index 050fb04eb..53ca5cdda 100644 --- a/README.md +++ b/README.md @@ -3,39 +3,38 @@ This repository contains the source code for .NET Portability Analyzer tools and dependencies. -|Branch|Build Status| -|---|---| -|master|[![][BuildStatus-Master]][myget]| -|dev|[![][BuildStatus-Dev]][myget]| +|Branch|Build Status +|---|---|---| +|master|[![Build Status](https://devdiv.visualstudio.com/DevDiv/_apis/build/status/CoreFxTools/dotnet-apiport-yaml?branchName=master)](https://devdiv.visualstudio.com/DevDiv/_build/latest?definitionId=12912&branchName=master) +|dev|[![Build Status](https://devdiv.visualstudio.com/DevDiv/_apis/build/status/CoreFxTools/dotnet-apiport-yaml?branchName=dev)](https://devdiv.visualstudio.com/DevDiv/_build/latest?definitionId=12912&branchName=dev) For a quick introduction, check out [this video on Channel 9][Channel 9 Video]: [][Channel 9 Video] -## Using this Repository +There is a Visual Studio extension available for VS 2017 and VS 2019: [.NET Portability Analyzer](https://marketplace.visualstudio.com/items?itemName=ConnieYau.NETPortabilityAnalyzer) -### Windows -There is a Visual Studio extension available for VS2017: [.NET Portability Analyzer](https://marketplace.visualstudio.com/items?itemName=ConnieYau.NETPortabilityAnalyzer) +## Using this Repository -Download and build for yourself: -1. Install [Visual Studio 2017 with .NET Core Workload][Visual Studio 2017] -2. Building: - * Visual Studio: `PortabilityTools.sln` - * Powershell: `.\build.ps1 -Configuration Debug -Platform AnyCPU` +See our [contributing guide](CONTRIBUTING.md) for instructions to +build and run from the source code in this repo. -### Linux/Mac +Sample usage to run the analysis from the command line: -1. Install [.NET Core SDK](https://dotnet.microsoft.com/download) -2. Execute: `build.sh` -3. Go to: `bin/Debug/ApiPort/netcoreapp2.1` -4. Run ApiPort by executing: `dotnet ApiPort.dll` - * Example: `dotnet ApiPort.dll listTargets` - * Example: `dotnet ApiPort.dll analyze -f Foo.dll -r HTML` -5. For convenience, create an alias command adding the following to your `~/.bash_profile`. Replace `{dotnet-apiport-folder}` with the path where you cloned the repo. +```ps1 +./init.ps1 +dotnet build src/ApiPort/ApiPort/ApiPort.csproj +dotnet bin/Debug/ApiPort/netcoreapp2.1/ApiPort.dll -- listTargets +dotnet bin/Debug/ApiPort/netcoreapp2.1/ApiPort.dll -- analyze -f Foo.dll -r HTML ``` + +If using bash for your shell, for convenience you may create an alias command adding the following to your `~/.bash_profile`. Replace `{dotnet-apiport-folder}` with the path where you cloned the repo. + +```bash alias apiport="dotnet {dotnet-apiport-folder}/bin/Debug/ApiPort/netcoreapp2.1/ApiPort.dll" ``` -This will alow you to use apiport globally from the command line: `apiport analyze -f Foo.dll -r HTML` + +This will allow you to use apiport globally from the command line: `apiport analyze -f Foo.dll -r HTML` ## Documentation @@ -43,7 +42,7 @@ This will alow you to use apiport globally from the command line: `apiport analy * [Platform Portability](docs/HowTo/PlatformPortability.md) * [Breaking Changes](docs/HowTo/BreakingChanges.md) * [.NET Portability Analyzer (Console application)](docs/Console) - * [.NET Core application](docs/Console/README.md#using-net-core-application) + * [.NET Core application](docs/Console/README.md#using-net-core-application) * [.NET Portability Analyzer (Visual Studio extension)](docs/VSExtension) ## Projects @@ -93,8 +92,6 @@ For an overview of all the .NET related projects, have a look at the This project is licensed under the [MIT license](LICENSE). -[BuildStatus-Master]: https://devdiv.visualstudio.com/_apis/public/build/definitions/0bdbc590-a062-4c3f-b0f6-9383f67865ee/484/badge -[BuildStatus-Dev]: https://devdiv.visualstudio.com/_apis/public/build/definitions/0bdbc590-a062-4c3f-b0f6-9383f67865ee/7913/badge [Channel 9 Video]: https://channel9.msdn.com/Blogs/Seth-Juarez/A-Brief-Look-at-the-NET-Portability-Analyzer [Contributing Guide]: https://github.com/dotnet/corefx/wiki/Contributing [Developer Guide]: https://github.com/dotnet/corefx/wiki/Developer-Guide @@ -105,5 +102,4 @@ This project is licensed under the [MIT license](LICENSE). [PR-Open]: https://github.com/Microsoft/dotnet-apiport/pulls?q=is%3Aopen+is%3Apr [myget]: https://dotnet.myget.org/gallery/dotnet-apiport [System.Reflection.Metadata]: https://github.com/dotnet/corefx/tree/master/src/System.Reflection.Metadata -[Visual Studio 2017]: https://docs.microsoft.com/dotnet/core/install/sdk?pivots=os-windows#install-with-visual-studio [VSIX Gallery]: http://vsixgallery.com/extension/55d15546-28ca-40dc-af23-dfa503e9c5fe diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fadddec29..6f6a06ad5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: - checkout: self clean: true - - powershell: .\init.ps1 + - powershell: .\init.ps1 -UpgradePrerequisites -NoRestore displayName: init.ps1 script - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 @@ -195,6 +195,7 @@ jobs: tests/**/*[Tt]ests/*.csproj !tests/**/*/ApiPortVS.Tests.csproj arguments: '--configuration $(BuildConfiguration) --framework $(BuildNetCoreAppTarget) --no-build' + enabled: false # it wasn't testing anything anyway. - task: DotNetCoreCLI@2 displayName: dotnet publish diff --git a/azure-pipelines/Get-NuGetTool.ps1 b/azure-pipelines/Get-NuGetTool.ps1 new file mode 100644 index 000000000..4431adb91 --- /dev/null +++ b/azure-pipelines/Get-NuGetTool.ps1 @@ -0,0 +1,22 @@ +<# +.SYNOPSIS + Downloads the NuGet.exe tool and returns the path to it. +.PARAMETER NuGetVersion + The version of the NuGet tool to acquire. +#> +Param( + [Parameter()] + [string]$NuGetVersion='5.2.0' +) + +$toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" +$binaryToolsPath = Join-Path $toolsPath $NuGetVersion +if (!(Test-Path $binaryToolsPath)) { $null = mkdir $binaryToolsPath } +$nugetPath = Join-Path $binaryToolsPath nuget.exe + +if (!(Test-Path $nugetPath)) { + Write-Host "Downloading nuget.exe $NuGetVersion..." -ForegroundColor Yellow + (New-Object System.Net.WebClient).DownloadFile("https://dist.nuget.org/win-x86-commandline/v$NuGetVersion/NuGet.exe", $nugetPath) +} + +return (Resolve-Path $nugetPath).Path diff --git a/azure-pipelines/Get-ProcDump.ps1 b/azure-pipelines/Get-ProcDump.ps1 new file mode 100644 index 000000000..1493fe4b2 --- /dev/null +++ b/azure-pipelines/Get-ProcDump.ps1 @@ -0,0 +1,14 @@ +<# +.SYNOPSIS +Downloads 32-bit and 64-bit procdump executables and returns the path to where they were installed. +#> +$version = '0.0.1' +$baseDir = "$PSScriptRoot\..\obj\tools" +$procDumpToolPath = "$baseDir\procdump.$version\bin" +if (-not (Test-Path $procDumpToolPath)) { + if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } + $baseDir = (Resolve-Path $baseDir).Path # Normalize it + & (& $PSScriptRoot\Get-NuGetTool.ps1) install procdump -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://api.nuget.org/v3/index.json | Out-Null +} + +(Resolve-Path $procDumpToolPath).Path diff --git a/azure-pipelines/Get-TempToolsPath.ps1 b/azure-pipelines/Get-TempToolsPath.ps1 new file mode 100644 index 000000000..bb3da8e33 --- /dev/null +++ b/azure-pipelines/Get-TempToolsPath.ps1 @@ -0,0 +1,13 @@ +if ($env:AGENT_TEMPDIRECTORY) { + $path = "$env:AGENT_TEMPDIRECTORY\$env:BUILD_BUILDID" +} elseif ($env:localappdata) { + $path = "$env:localappdata\gitrepos\tools" +} else { + $path = "$PSScriptRoot\..\obj\tools" +} + +if (!(Test-Path $path)) { + New-Item -ItemType Directory -Path $Path | Out-Null +} + +(Resolve-Path $path).Path diff --git a/azure-pipelines/Set-EnvVars.ps1 b/azure-pipelines/Set-EnvVars.ps1 new file mode 100644 index 000000000..9d14d9aa0 --- /dev/null +++ b/azure-pipelines/Set-EnvVars.ps1 @@ -0,0 +1,79 @@ +<# +.SYNOPSIS + Set environment variables in the environment. + Azure Pipeline and CMD environments are considered. +.PARAMETER Variables + A hashtable of variables to be set. +.OUTPUTS + A boolean indicating whether the environment variables can be expected to propagate to the caller's environment. +#> +[CmdletBinding(SupportsShouldProcess=$true)] +Param( + [Parameter(Mandatory=$true, Position=1)] + $Variables, + [string[]]$PrependPath +) + +if ($Variables.Count -eq 0) { + return $true +} + +$cmdInstructions = !$env:TF_BUILD -and !$env:GITHUB_ACTIONS -and $env:PS1UnderCmd -eq '1' +if ($cmdInstructions) { + Write-Warning "Environment variables have been set that will be lost because you're running under cmd.exe" + Write-Host "Environment variables that must be set manually:" -ForegroundColor Blue +} else { + Write-Host "Environment variables set:" -ForegroundColor Blue + Write-Host ($Variables | Out-String) + if ($PrependPath) { + Write-Host "Paths prepended to PATH: $PrependPath" + } +} + +if ($env:TF_BUILD) { + Write-Host "Azure Pipelines detected. Logging commands will be used to propagate environment variables and prepend path." +} + +if ($env:GITHUB_ACTIONS) { + Write-Host "GitHub Actions detected. Logging commands will be used to propagate environment variables and prepend path." +} + +$Variables.GetEnumerator() |% { + Set-Item -Path env:$($_.Key) -Value $_.Value + + # If we're running in a cloud CI, set these environment variables so they propagate. + if ($env:TF_BUILD) { + Write-Host "##vso[task.setvariable variable=$($_.Key);]$($_.Value)" + } + if ($env:GITHUB_ACTIONS) { + Write-Host "::set-env name=$($_.Key)::$($_.Value)" + } + + if ($cmdInstructions) { + Write-Host "SET $($_.Key)=$($_.Value)" + } +} + +$pathDelimiter = ';' +if ($IsMacOS -or $IsLinux) { + $pathDelimiter = ':' +} + +if ($PrependPath) { + $PrependPath |% { + $newPathValue = "$_$pathDelimiter$env:PATH" + Set-Item -Path env:PATH -Value $newPathValue + if ($cmdInstructions) { + Write-Host "SET PATH=$newPathValue" + } + + if ($env:TF_BUILD) { + Write-Host "##vso[task.prependpath]$_" + } + if ($env:GITHUB_ACTIONS) { + Write-Host "::add-path::$_" + } + } +} + +return !$cmdInstructions diff --git a/azure-pipelines/variables/DotNetSdkVersion.ps1 b/azure-pipelines/variables/DotNetSdkVersion.ps1 new file mode 100644 index 000000000..b213fbc27 --- /dev/null +++ b/azure-pipelines/variables/DotNetSdkVersion.ps1 @@ -0,0 +1,2 @@ +$globalJson = Get-Content -Path "$PSScriptRoot\..\..\global.json" | ConvertFrom-Json +$globalJson.sdk.version diff --git a/build.sh b/build.sh deleted file mode 100755 index 440b90c31..000000000 --- a/build.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash -set -e - -export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true -export HOME=~ -export NUGET_PACKAGES=~/.nuget/packages -export NUGET_HTTP_CACHE_PATH=~/.local/share/NuGet/v3-cache - -Configuration=Debug - -RootDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DotNetSDKPath=$RootDir"/.tools/dotnet/"$DotNetSDKVersion -DotNetExe=$DotNetSDKPath"/dotnet" - -BuildApps=true -RunTests=false - -usage() { - echo "Usage: build.sh [-c|--configuration ] [--downloadCatalog] [--runTests] [--no-build]" -} - -downloadCatalog() { - local isForce=$1 - local catalog=$RootDir"/.data/catalog.bin" - local data=$(dirname $catalog) - - if [[ ! -e $data ]]; then - mkdir $data - fi - - if [[ $isForce == "true" && -e $catalog ]]; then - echo "Deleting existing catalog" - rm $catalog - fi - - if [[ ! -e $catalog ]]; then - echo "Downloading catalog.bin..." - curl --output $catalog "https://portability.blob.core.windows.net/catalog/catalog.bin" - fi -} - -installSDK() { - local dotnetPath=$(command -v dotnet) - - if [[ $dotnetPath != "" ]]; then - echo "dotnet is found on PATH. using that." - DotNetExe=$dotnetPath - return 0 - fi - - if [[ -e $DotNetExe ]]; then - echo $DotNetExe" exists. Skipping install..." - return 0 - fi - - local DotNetToolsPath=$(dirname $DotNetSDKPath) - - if [ ! -d $DotNetToolsPath ]; then - mkdir -p $DotNetToolsPath - fi - - curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel Current --install-dir $DotNetSDKPath -} - -tryGetTargetFramework() { - local file=$1 - local targetFramework=$(awk -F: '/.*netcoreapp[1-9]\.[0-9].*<\/TargetFramework(s)?>/ { print $0 }' $file | sed 's/.*\(netcoreapp[1-9]\.[0-9]\).*/\1/' | tail -n 1) - echo $targetFramework -} - -build() { - echo "Building ApiPort... Configuration: ["$Configuration"]" - - pushd src/ApiPort/ApiPort >/dev/null - - local targetFramework=$(tryGetTargetFramework ApiPort.props) - echo "Building for: $targetFramework" - - $DotNetExe build ApiPort.csproj -f $targetFramework -c $Configuration - $DotNetExe build ApiPort.Offline.csproj -f $targetFramework -c $Configuration - popd >/dev/null -} - -runTest() { - ls $1/*.csproj | while read file; do - local targetFramework=$(tryGetTargetFramework $file) - - if [[ $targetFramework == "" ]]; then - echo "Skipping "$file - echo "--- Desktop .NET Framework testing is not currently supported on Unix." - else - echo "Testing "$file - $DotNetExe test $file -c $Configuration --logger trx --framework $targetFramework --results-directory $2 - fi - done -} - -findAndRunTests() { - local testResultsDirectory=$COMMON_TESTRESULTSDIRECTORY - - if [[ $testResultsDirectory == "" ]]; then - testResultsDirectory=$RootDir/TestResults - echo "Results directory not specified, using $testResultsDirectory." - else - echo "Using common one set by build agent." - fi - - find tests/ -type d -name "*\.Tests" | while read file; do - runTest $file $testResultsDirectory - done -} - -while [[ $# -gt 0 ]]; do - option="$(echo $1 | awk '{print tolower($0)}')" - case "$option" in - "-?" | "--help") - usage - exit 1 - ;; - "-c" | "--configuration") - Configuration="$2" - shift 2 - ;; - "--runtests") - RunTests=true - shift - ;; - "--no-build") - BuildApps=false - shift - ;; - "--downloadcatalog") - downloadCatalog "true" - exit 0 - ;; - *) - echo "Unknown option: "$option - usage - exit 1 - ;; - esac -done - -# Enable insensitive case-matching -shopt -s nocasematch - -if [[ "$Configuration" != "Debug" && "$Configuration" != "Release" ]]; then - echo "ERROR: Supported configuration types are Debug or Release. Invalid configuration: "$Configuration - usage - exit 3 -fi - -shopt -u nocasematch - -installSDK - -if [[ ! -e $DotNetExe ]]; then - echo "ERROR: It should have been installed from build/dotnet-install.sh" - exit 2 -fi - -if [[ $BuildApps == "true" ]]; then - downloadCatalog - build -fi - -if [[ $RunTests == "true" ]]; then - if [[$BuildApps != "true" ]]; then - echo "WARNING: Not building applications because --no-build is set." - fi - - findAndRunTests -fi - -echo "Finished!" diff --git a/init.cmd b/init.cmd new file mode 100644 index 000000000..970285c2f --- /dev/null +++ b/init.cmd @@ -0,0 +1,4 @@ +@echo off +SETLOCAL +set PS1UnderCmd=1 +powershell.exe -NoProfile -NoLogo -ExecutionPolicy bypass -Command "try { & '%~dpn0.ps1' %*; exit $LASTEXITCODE } catch { write-host $_; exit 1 }" diff --git a/init.ps1 b/init.ps1 old mode 100644 new mode 100755 index 341bd8b83..42c5a2fce --- a/init.ps1 +++ b/init.ps1 @@ -1,31 +1,81 @@ -$ErrorActionPreference = "Stop" +#!/usr/bin/env pwsh -function DownloadFile($url, $outputPath) { - Write-Host "Attempt to download to $outputPath" +<# +.SYNOPSIS +Installs dependencies required to build and test the projects in this repository. +.DESCRIPTION +This MAY not require elevation, as the SDK and runtimes are installed to a per-user location, +unless the `-InstallLocality` switch is specified directing to a per-repo or per-machine location. +See detailed help on that switch for more information. +.PARAMETER InstallLocality +A value indicating whether dependencies should be installed locally to the repo or at a per-user location. +Per-user allows sharing the installed dependencies across repositories and allows use of a shared expanded package cache. +Visual Studio will only notice and use these SDKs/runtimes if VS is launched from the environment that runs this script. +Per-repo allows for high isolation, allowing for a more precise recreation of the environment within an Azure Pipelines build. +When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. +Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. +Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +.PARAMETER NoPrerequisites +Skips the installation of prerequisite software (e.g. SDKs, tools). +.PARAMETER UpgradePrerequisites +Takes time to install prerequisites even if they are already present in case they need to be upgraded. +No effect if -NoPrerequisites is specified. +.PARAMETER NoRestore +Skips the package restore step. +.PARAMETER AccessToken +An optional access token for authenticating to Azure Artifacts authenticated feeds. +#> +[CmdletBinding(SupportsShouldProcess=$true)] +Param ( + [ValidateSet('repo','user','machine')] + [string]$InstallLocality='user', + [Parameter()] + [switch]$NoPrerequisites, + [Parameter()] + [switch]$UpgradePrerequisites, + [Parameter()] + [switch]$NoRestore, + [Parameter()] + [string]$AccessToken +) - # If the file has been downloaded don't download again. An empty file implies a failed download - if(Test-Path $outputPath) { - $file = Get-ChildItem $outputPath +$EnvVars = @{} - if($file.Length -gt 0) { - Write-Host "$outputPath is already downloaded" - return; - } - } +if (!$NoPrerequisites) { + & "$PSScriptRoot\tools\Install-NuGetCredProvider.ps1" -AccessToken $AccessToken -Force:$UpgradePrerequisites + & "$PSScriptRoot\tools\Install-DotNetSdk.ps1" -InstallLocality $InstallLocality - try { - # Create placeholder so directory exists - New-Item -Type File $OutputPath -Force | Out-Null + # The procdump tool and env var is required for dotnet test to collect hang/crash dumps of tests. + # But it only works on Windows. + if ($env:OS -eq 'Windows_NT') { + $EnvVars['PROCDUMP_PATH'] = & "$PSScriptRoot\azure-pipelines\Get-ProcDump.ps1" + } +} - # Attempt to download. If fails, placeholder remains so msbuild won't complain - Invoke-WebRequest $url -OutFile $OutputPath | Out-Null +& "$PSScriptRoot\tools\Get-CatalogBin.ps1" - Write-Host "Downloaded $OutputPath" - } catch { - Write-Error "Failed to download '$url'. $($Error[0])" - } -} +# Workaround nuget credential provider bug that causes very unreliable package restores on Azure Pipelines +$env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 +$env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 + +Push-Location $PSScriptRoot +try { + $HeaderColor = 'Green' -$address = "https://portability.blob.core.windows.net/catalog/catalog.bin" + if (!$NoRestore -and $PSCmdlet.ShouldProcess("NuGet packages", "Restore")) { + Write-Host "Restoring NuGet packages" -ForegroundColor $HeaderColor + dotnet restore + if ($lastexitcode -ne 0) { + throw "Failure while restoring packages." + } + } -DownloadFile "$address" "$PSScriptRoot\.data\catalog.bin" + & "$PSScriptRoot\azure-pipelines\Set-EnvVars.ps1" -Variables $EnvVars | Out-Null +} +catch { + Write-Error $error[0] + exit $lastexitcode +} +finally { + Pop-Location +} diff --git a/tools/Get-CatalogBin.ps1 b/tools/Get-CatalogBin.ps1 new file mode 100644 index 000000000..b998c52af --- /dev/null +++ b/tools/Get-CatalogBin.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = "Stop" + +function DownloadFile($url, $outputPath) { + Write-Host "Attempt to download to $outputPath" + + # If the file has been downloaded don't download again. An empty file implies a failed download + if(Test-Path $outputPath) { + $file = Get-ChildItem $outputPath + + if($file.Length -gt 0) { + Write-Host "$outputPath is already downloaded" + return; + } + } + + try { + # Create placeholder so directory exists + New-Item -Type File $OutputPath -Force | Out-Null + + # Attempt to download. If fails, placeholder remains so msbuild won't complain + Invoke-WebRequest $url -OutFile $OutputPath | Out-Null + + Write-Host "Downloaded $OutputPath" + } catch { + Write-Error "Failed to download '$url'. $($Error[0])" + } +} + +$address = "https://portability.blob.core.windows.net/catalog/catalog.bin" + +DownloadFile "$address" "$PSScriptRoot\..\.data\catalog.bin" diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 new file mode 100644 index 000000000..2a141327d --- /dev/null +++ b/tools/Install-DotNetSdk.ps1 @@ -0,0 +1,159 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Installs the .NET SDK specified in the global.json file at the root of this repository, +along with supporting .NET Core runtimes used for testing. +.DESCRIPTION +This MAY not require elevation, as the SDK and runtimes are installed locally to this repo location, +unless `-InstallLocality machine` is specified. +.PARAMETER InstallLocality +A value indicating whether dependencies should be installed locally to the repo or at a per-user location. +Per-user allows sharing the installed dependencies across repositories and allows use of a shared expanded package cache. +Visual Studio will only notice and use these SDKs/runtimes if VS is launched from the environment that runs this script. +Per-repo allows for high isolation, allowing for a more precise recreation of the environment within an Azure Pipelines build. +When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. +Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. +Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +#> +[CmdletBinding(SupportsShouldProcess=$true,ConfirmImpact='Medium')] +Param ( + [ValidateSet('repo','user','machine')] + [string]$InstallLocality='user' +) + +$DotNetInstallScriptRoot = "$PSScriptRoot/../obj/tools" +if (!(Test-Path $DotNetInstallScriptRoot)) { New-Item -ItemType Directory -Path $DotNetInstallScriptRoot -WhatIf:$false | Out-Null } +$DotNetInstallScriptRoot = Resolve-Path $DotNetInstallScriptRoot + +# Look up actual required .NET Core SDK version from global.json +$sdkVersion = & "$PSScriptRoot/../azure-pipelines/variables/DotNetSdkVersion.ps1" + +# Search for all .NET Core runtime versions referenced from MSBuild projects and arrange to install them. +$runtimeVersions = @() +Get-ChildItem "$PSScriptRoot\..\src\*.*proj","$PSScriptRoot\..\tests\*.*proj","$PSScriptRoot\..\Directory.Build.props" -Recurse |% { + $projXml = [xml](Get-Content -Path $_) + $targetFrameworks = $projXml.Project.PropertyGroup.TargetFramework + if (!$targetFrameworks) { + $targetFrameworks = $projXml.Project.PropertyGroup.TargetFrameworks + if ($targetFrameworks) { + $targetFrameworks = $targetFrameworks -Split ';' + } + } + $targetFrameworks |? { $_ -match 'netcoreapp(\d+\.\d+)' } |% { + $runtimeVersions += $Matches[1] + } +} + +Function Get-FileFromWeb([Uri]$Uri, $OutDir) { + $OutFile = Join-Path $OutDir $Uri.Segments[-1] + if (!(Test-Path $OutFile)) { + Write-Verbose "Downloading $Uri..." + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) + } finally { + # This try/finally causes the script to abort + } + } + + $OutFile +} + +Function Get-InstallerExe($Version, [switch]$Runtime) { + $sdkOrRuntime = 'Sdk' + if ($Runtime) { $sdkOrRuntime = 'Runtime' } + + # Get the latest/actual version for the specified one + if (([Version]$Version).Build -eq -1) { + $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/latest.version" -UseBasicParsing) + $Version = $versionInfo[-1] + } + + Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/dotnet-$($sdkOrRuntime.ToLowerInvariant())-$Version-win-x64.exe" -OutDir "$DotNetInstallScriptRoot" +} + +Function Install-DotNet($Version, [switch]$Runtime) { + if ($Runtime) { $sdkSubstring = '' } else { $sdkSubstring = 'SDK ' } + Write-Host "Downloading .NET Core $sdkSubstring$Version..." + $Installer = Get-InstallerExe -Version $Version -Runtime:$Runtime + Write-Host "Installing .NET Core $sdkSubstring$Version..." + cmd /c start /wait $Installer /install /quiet + if ($LASTEXITCODE -ne 0) { + throw "Failure to install .NET Core SDK" + } +} + +$switches = @( + '-Architecture','x64' +) +$envVars = @{ + # For locally installed dotnet, skip first time experience which takes a long time + 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' = 'true'; +} + +if ($InstallLocality -eq 'machine') { + if ($IsWindows) { + if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { + Install-DotNet -Version $sdkVersion + } + + $runtimeVersions | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + Install-DotNet -Version $_ -Runtime + } + } + + return + } else { + $DotNetInstallDir = '/usr/share/dotnet' + } +} elseif ($InstallLocality -eq 'repo') { + $DotNetInstallDir = "$DotNetInstallScriptRoot/.dotnet" +} elseif ($env:AGENT_TOOLSDIRECTORY) { + $DotNetInstallDir = "$env:AGENT_TOOLSDIRECTORY/dotnet" +} else { + $DotNetInstallDir = Join-Path $HOME .dotnet +} + +Write-Host "Installing .NET Core SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue + +if ($DotNetInstallDir) { + $switches += '-InstallDir',$DotNetInstallDir + $envVars['DOTNET_MULTILEVEL_LOOKUP'] = '0' + $envVars['DOTNET_ROOT'] = $DotNetInstallDir +} + +if ($IsMacOS -or $IsLinux) { + $DownloadUri = "https://dot.net/v1/dotnet-install.sh" + $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" +} else { + $DownloadUri = "https://dot.net/v1/dotnet-install.ps1" + $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.ps1" +} + +if (-not (Test-Path $DotNetInstallScriptPath)) { + Invoke-WebRequest -Uri $DownloadUri -OutFile $DotNetInstallScriptPath -UseBasicParsing + if ($IsMacOS -or $IsLinux) { + chmod +x $DotNetInstallScriptPath + } +} + +if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { + Invoke-Expression -Command "$DotNetInstallScriptPath -Version $sdkVersion $switches" +} else { + Invoke-Expression -Command "$DotNetInstallScriptPath -Version $sdkVersion $switches -DryRun" +} + +$switches += '-Runtime','dotnet' + +$runtimeVersions | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + Invoke-Expression -Command "$DotNetInstallScriptPath -Channel $_ $switches" + } else { + Invoke-Expression -Command "$DotNetInstallScriptPath -Channel $_ $switches -DryRun" + } +} + +if ($PSCmdlet.ShouldProcess("Set DOTNET environment variables to discover these installed runtimes?")) { + & "$PSScriptRoot/../azure-pipelines/Set-EnvVars.ps1" -Variables $envVars -PrependPath $DotNetInstallDir | Out-Null +} diff --git a/tools/Install-NuGetCredProvider.ps1 b/tools/Install-NuGetCredProvider.ps1 new file mode 100644 index 000000000..2b3fb6fb4 --- /dev/null +++ b/tools/Install-NuGetCredProvider.ps1 @@ -0,0 +1,74 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Downloads and installs the Microsoft Artifacts Credential Provider + from https://github.com/microsoft/artifacts-credprovider + to assist in authenticating to Azure Artifact feeds in interactive development + or unattended build agents. +.PARAMETER Force + Forces install of the CredProvider plugin even if one already exists. This is useful to upgrade an older version. +.PARAMETER AccessToken + An optional access token for authenticating to Azure Artifacts authenticated feeds. +#> +[CmdletBinding()] +Param ( + [Parameter()] + [switch]$Force, + [Parameter()] + [string]$AccessToken +) + +$toolsPath = & "$PSScriptRoot\..\azure-pipelines\Get-TempToolsPath.ps1" + +if ($IsMacOS -or $IsLinux) { + $installerScript = "installcredprovider.sh" + $sourceUrl = "https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" +} else { + $installerScript = "installcredprovider.ps1" + $sourceUrl = "https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1" +} + +$installerScript = Join-Path $toolsPath $installerScript + +if (!(Test-Path $installerScript)) { + Invoke-WebRequest $sourceUrl -OutFile $installerScript +} + +$installerScript = (Resolve-Path $installerScript).Path + +if ($IsMacOS -or $IsLinux) { + chmod u+x $installerScript +} + +& $installerScript -Force:$Force + +if ($AccessToken) { + $endpoints = @() + + $endpointURIs = @() + Get-ChildItem "$PSScriptRoot\..\nuget.config" -Recurse |% { + $nugetConfig = [xml](Get-Content -Path $_) + + $nugetConfig.configuration.packageSources.add |? { ($_.value -match '^https://pkgs\.dev\.azure\.com/') -or ($_.value -match '^https://[\w\-]+\.pkgs\.visualstudio\.com/') } |% { + if ($endpointURIs -notcontains $_.Value) { + $endpointURIs += $_.Value + $endpoint = New-Object -TypeName PSObject + Add-Member -InputObject $endpoint -MemberType NoteProperty -Name endpoint -Value $_.value + Add-Member -InputObject $endpoint -MemberType NoteProperty -Name username -Value ado + Add-Member -InputObject $endpoint -MemberType NoteProperty -Name password -Value $AccessToken + $endpoints += $endpoint + } + } + } + + $auth = New-Object -TypeName PSObject + Add-Member -InputObject $auth -MemberType NoteProperty -Name endpointCredentials -Value $endpoints + + $authJson = ConvertTo-Json -InputObject $auth + $envVars = @{ + 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS'=$authJson; + } + + & "$PSScriptRoot\..\azure-pipelines\Set-EnvVars.ps1" -Variables $envVars | Out-Null +}