[ci] Move .NET SDK installation to xaprepare

pjcollins · pjcollins · commit f6dc8fbc55c6 · 2020-10-07T17:30:31.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@ Configuration.OperatingSystem.props
 obj
 packages
 .DS_Store
+.dotnet
 .nuget
 *.nupkg
 TestResult.xml
diff --git a/build-tools/automation/azure-pipelines-oss.yaml b/build-tools/automation/azure-pipelines-oss.yaml
@@ -26,12 +26,8 @@ variables:
   InstallerArtifactName: Installers
   EXTRA_MSBUILD_ARGS: /p:AutoProvision=True /p:AutoProvisionUsesSudo=True /p:IgnoreMaxMonoVersion=False
   PREPARE_FLAGS: PREPARE_CI=1 PREPARE_CI_PR=1
-  DotNetCoreVersion: 3.1.201
-  # Version number from: https://github.com/dotnet/installer#installers-and-binaries
-  DotNetCorePreviewVersion: 5.0.100-rc.2.20459.1
   GitHub.Token: $(github--pat--vs-mobiletools-engineering-service2)
 
-
 stages:
 - stage: mac_stage
   displayName: Mac
@@ -52,15 +48,6 @@ stages:
       clean: true             # Executes: git clean -ffdx && git reset --hard HEAD
       submodules: recursive
 
-    - template: yaml-templates/use-dot-net.yaml
-      parameters:
-        version: $(DotNetCorePreviewVersion)
-        remove_dotnet: true
-
-    - template: yaml-templates/use-dot-net.yaml
-      parameters:
-        version: $(DotNetCoreVersion)
-
     - bash: |
         keychains=`security list-keychains`
         if [[ "$keychains" =~ "\"/Users/${USER}/Library/Keychains/builder.keychain-db\"" ]]; then
@@ -90,6 +77,9 @@ stages:
         fi
       displayName: restore default keychain
 
+    - script: echo "##vso[task.setvariable variable=PATH]$(Build.SourcesDirectory)/.dotnet:$PATH"
+      displayName: add local dotnet installation to PATH
+
     # Prepare and build everything
     - script: >
         echo "make prepare-update-mono V=1 CONFIGURATION=$(XA.Build.Configuration) $(PREPARE_FLAGS) MSBUILD_ARGS='$(EXTRA_MSBUILD_ARGS)'" &&
diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml
@@ -51,9 +51,6 @@ variables:
   InstallerArtifactName: installers
   TestAssembliesArtifactName: test-assemblies
   NUnitConsoleVersion: 3.11.1
-  DotNetCoreVersion: 3.1.201
-  # Version number from: https://github.com/dotnet/installer#installers-and-binaries
-  DotNetCorePreviewVersion: 5.0.100-rc.2.20480.7
   HostedMacMojave: Hosted Mac Internal Mojave
   HostedMac: Hosted Mac Internal
   HostedWinVS2019: Hosted Windows 2019 with VS2019
@@ -137,15 +134,6 @@ stages:
     - script: echo "##vso[task.setvariable variable=JI_JAVA_HOME]$HOME/Library/Android/$(XA.Jdk11.Folder)"
       displayName: set JI_JAVA_HOME
 
-    - template: yaml-templates/use-dot-net.yaml
-      parameters:
-        version: $(DotNetCorePreviewVersion)
-        remove_dotnet: true
-
-    - template: yaml-templates/use-dot-net.yaml
-      parameters:
-        version: $(DotNetCoreVersion)
-
     - template: install-certificates.yml@yaml
       parameters:
         DeveloperIdApplication: $(developer-id-application)
@@ -161,6 +149,9 @@ stages:
         provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/provisioning/xcode.csx
         provisioning_extra_args: '-v -v -v -v'
 
+    - script: echo "##vso[task.setvariable variable=PATH]$(System.DefaultWorkingDirectory)/xamarin-android/.dotnet:$PATH"
+      displayName: add local dotnet installation to PATH
+
     # Prepare and build everything
     - script: make prepare-update-mono V=1 CONFIGURATION=$(XA.Build.Configuration) PREPARE_CI=1 PREPARE_AUTOPROVISION=1
       workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
@@ -287,15 +278,6 @@ stages:
         echo ##vso[task.setvariable variable=JI_JAVA_HOME]%USERPROFILE%\android-toolchain\$(XA.Jdk11.Folder)
       displayName: set JI_JAVA_HOME
 
-    - template: yaml-templates\use-dot-net.yaml
-      parameters:
-        version: $(DotNetCorePreviewVersion)
-        remove_dotnet: true
-
-    - template: yaml-templates\use-dot-net.yaml
-      parameters:
-        version: $(DotNetCoreVersion)
-
     # Downgrade the XA .vsix installed into the instance of VS that we are building with so that we don't restore/build against a test version.
     # The VS installer will attempt to resume any failed or partial installation before trying to downgrade Xamarin.Android.
     # VSIXInstaller.exe will exit non-zero when the downgrade attempt is a no-op, so we will allow this step to fail silently.
@@ -323,6 +305,10 @@ stages:
 
     - template: environment/win/vs-msbuild.v1.yml@yaml   # Display (in log) VS installation(s) including installation status with associated MSBuild location(s)
 
+    - script: |
+        echo ##vso[task.setvariable variable=PATH]$(Build.SourcesDirectory)\.dotnet;%PATH%
+      displayName: add local dotnet installation to PATH
+
     - task: MSBuild@1
       displayName: msbuild Xamarin.Android /t:Prepare
       inputs:
diff --git a/build-tools/automation/yaml-templates/setup-test-environment.yaml b/build-tools/automation/yaml-templates/setup-test-environment.yaml
@@ -21,15 +21,6 @@ steps:
   displayName: set JI_JAVA_HOME
   condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT'))
 
-- template: use-dot-net.yaml
-  parameters:
-    version: $(DotNetCorePreviewVersion)
-    remove_dotnet: true
-
-- template: use-dot-net.yaml
-  parameters:
-    version: $(DotNetCoreVersion)
-
 - script: |
     dotnet tool install --global boots
   displayName: install boots
diff --git a/build-tools/automation/yaml-templates/use-dot-net.yaml b/build-tools/automation/yaml-templates/use-dot-net.yaml
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs
@@ -15,5 +15,10 @@ partial class Paths
 		{
 			public const string NdkToolchainOSTag = "linux-x86_64";
 		}
+
+		partial class Urls
+		{
+			public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.sh");
+		}
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs
@@ -21,5 +21,10 @@ partial class Paths
 			public static readonly string UnstrippedLibMonoSgenName = $"{LibMonoSgenBaseName}.d{Defaults.NativeLibraryExtension}";
 			public static readonly string StrippedLibMonoSgenName = $"{LibMonoSgenBaseName}{Defaults.NativeLibraryExtension}";
 		}
+
+		partial class Urls
+		{
+			public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.sh");
+		}
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs
@@ -19,5 +19,10 @@ partial class Paths
 			public static readonly string MonoRuntimeHostMingwNativeLibraryPrefix = Path.Combine ("..", "bin");
 			public const string NdkToolchainOSTag                                 = "windows-x86_64";
 		}
+
+		partial class Urls
+		{
+			public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.ps1");
+		}
 	}
 }
diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs
@@ -64,6 +64,10 @@ public static partial class Defaults
 			public static readonly Version JetBrainsOpenJDK8Version = new Version (Configurables.JetBrainsOpenJDK8VersionFull);
 			public static readonly Version JetBrainsOpenJDK8Release = new Version (Configurables.JetBrainsOpenJDK8Release);
 
+			public const string DotNetVersion = "3.1.201";
+			// Version number from: https://github.com/dotnet/installer#installers-and-binaries
+			public const string DotNetPreviewVersion = "5.0.100-rc.2.20480.7";
+
 			// Mono runtimes
 			public const string DebugFileExtension                         = ".pdb";
 			public const string MonoHostMingwRuntimeNativeLibraryExtension = WindowsDLLSuffix;
@@ -274,6 +278,7 @@ public static partial class Paths
 			public static readonly string BuildToolsDir                    = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "build-tools");
 			public static readonly string BuildToolsScriptsDir             = Path.Combine (BuildToolsDir, "scripts");
 			public static readonly string BinDirRoot                       = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "bin");
+			public static readonly string DotNetRoot                       = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, ".dotnet");
 			public static readonly string ExternalDir                      = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "external");
 			public static readonly string LocalNugetPath                   = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, ".nuget", "NuGet.exe");
 			public static readonly string ExternalGitDepsFilePath          = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, ".external");
diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Required.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Required.cs
@@ -21,6 +21,7 @@ protected override void AddSteps (Context context)
 			// Install essential tools, should they be missing
 			context.SetCondition (KnownConditions.EnsureEssential, true);
 
+			Steps.Add (new Step_InstallDotNetSDKs ());
 			Steps.Add (new Step_GenerateFiles (atBuildStart: true, onlyRequired: true));
 			Steps.Add (new Step_PrepareExternalJavaInterop ());
 		}
diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs
@@ -19,6 +19,7 @@ protected override void AddSteps (Context context)
 				throw new ArgumentNullException (nameof (context));
 
 			Steps.Add (new Step_ShowEnabledRuntimes ());
+			Steps.Add (new Step_InstallDotNetSDKs ());
 			Steps.Add (new Step_InstallJetBrainsOpenJDK8 ());
 			Steps.Add (new Step_InstallJetBrainsOpenJDK11 ());
 			Steps.Add (new Step_Android_SDK_NDK ());
diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetSDKs.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetSDKs.cs
@@ -0,0 +1,108 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+
+namespace Xamarin.Android.Prepare
+{
+	class Step_InstallDotNetSDKs : StepWithDownloadProgress
+	{
+		public Step_InstallDotNetSDKs ()
+			: base ("Install required .NET Core SDKs locally")
+		{ }
+
+		protected override async Task<bool> Execute (Context context)
+		{
+			if (context == null)
+				throw new ArgumentNullException (nameof (context));
+
+			var dotnetPath = Configurables.Paths.DotNetRoot;
+			var dotnetVersion = Configurables.Defaults.DotNetVersion;
+			var dotnetPreviewVersion = Configurables.Defaults.DotNetPreviewVersion;
+
+			// Delete any custom Microsoft.Android packs that may have been installed by test runs. Other ref/runtime packs will be ignored.
+			var packsPath = Path.Combine (dotnetPath, "packs");
+			if (Directory.Exists (packsPath)) {
+				foreach (var packToRemove in Directory.EnumerateDirectories (packsPath).Where (p => new DirectoryInfo (p).Name.Contains ("Android")))
+					Utilities.DeleteDirectory (packToRemove);
+			}
+
+			// Delete any unnecessary SDKs if they exist.
+			var sdkPath = Path.Combine (dotnetPath, "sdk");
+			if (Directory.Exists (sdkPath)) {
+				var installedSdks = Directory.EnumerateDirectories (sdkPath).Select (d => new DirectoryInfo (d));
+				foreach (var sdkToRemove in installedSdks.Where (s => s.Name != dotnetVersion && s.Name != dotnetPreviewVersion))
+					Utilities.DeleteDirectory (sdkToRemove.FullName);
+			}
+
+			if (!await InstallDotNetAsync (context, dotnetPath, dotnetVersion)) {
+				Log.ErrorLine ($"Installation of dotnet SDK {dotnetVersion} failed.");
+				return false;
+			}
+
+			if (!await InstallDotNetAsync (context, dotnetPath, dotnetPreviewVersion)) {
+				Log.ErrorLine ($"Installation of dotnet SDK {dotnetPreviewVersion} failed.");
+				return false;
+			}
+
+			SetVariables (dotnetPath);
+			return true;
+		}
+
+		async Task<bool> InstallDotNetAsync (Context context, string dotnetPath, string version)
+		{
+			if (Directory.Exists (Path.Combine (dotnetPath, "sdk", version)))
+				return true;
+
+			Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript;
+			string dotnetScriptPath = Path.Combine (dotnetPath, Path.GetFileName (dotnetScriptUrl.LocalPath));
+
+			if (File.Exists (dotnetScriptPath))
+				Utilities.DeleteFile (dotnetScriptPath);
+			Log.StatusLine ("Downloading dotnet-install...");
+
+			(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl);
+			if (!success) {
+				if (status == HttpStatusCode.NotFound)
+					Log.ErrorLine ("dotnet-install URL not found");
+				else
+					Log.ErrorLine ("Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})");
+				return false;
+			}
+
+			DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
+			Log.StatusLine ($"  {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White);
+			await Download (context, dotnetScriptUrl, dotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus);
+
+			if (!File.Exists (dotnetScriptPath)) {
+				Log.ErrorLine ($"Download of dotnet-install from {dotnetScriptUrl} failed");
+				return false;
+			}
+
+			Log.StatusLine ($"Installing dotnet SDK '{version}'...");
+
+
+			if (Context.IsWindows) {
+				return Utilities.RunCommand ("powershell.exe", new string [] {
+					"-NoProfile", "-ExecutionPolicy", "unrestricted", "-file", dotnetScriptPath,
+					"-Version", version, "-InstallDir", dotnetPath, "-Verbose"
+				});
+			} else {
+				return Utilities.RunCommand ("bash", new string [] {
+					dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose"
+				});
+			}
+		}
+
+		void SetVariables (string dotnetPath)
+		{
+			var newPath = $"{dotnetPath}{Path.PathSeparator}{Environment.GetEnvironmentVariable ("PATH")}";
+			Environment.SetEnvironmentVariable ("PATH", newPath);
+			Environment.SetEnvironmentVariable ("DOTNET_ROOT", dotnetPath);
+			Log.MessageLine ($"##vso[task.setvariable variable=PATH]{newPath}");
+			Log.MessageLine ($"##vso[task.setvariable variable=DOTNET_ROOT]{dotnetPath}");
+		}
+
+	}
+}
diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj
@@ -146,6 +146,7 @@
     <Compile Include="Steps\Step_GenerateCGManifest.cs" />
     <Compile Include="Steps\Step_GenerateFiles.cs" />
     <Compile Include="Steps\Step_Get_Windows_Binutils.cs" />
+    <Compile Include="Steps\Step_InstallDotNetSDKs.cs" />
     <Compile Include="Steps\Step_InstallJetBrainsOpenJDK.cs" />
     <Compile Include="Steps\Step_InstallMonoRuntimes.cs" />
     <Compile Include="Steps\Step_PrepareExternal.cs" />

Original file line number	Diff line number	Diff line change
`@@ -15,5 +15,10 @@ partial class Paths`
`15`	`15`	`{`
`16`	`16`	`public const string NdkToolchainOSTag = "linux-x86_64";`
`17`	`17`	`}`
	`18`	`+`
	`19`	`+ partial class Urls`
	`20`	`+ {`
	`21`	`+ public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.sh");`
	`22`	`+ }`
`18`	`23`	`}`
`19`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,5 +21,10 @@ partial class Paths`
`21`	`21`	`public static readonly string UnstrippedLibMonoSgenName = $"{LibMonoSgenBaseName}.d{Defaults.NativeLibraryExtension}";`
`22`	`22`	`public static readonly string StrippedLibMonoSgenName = $"{LibMonoSgenBaseName}{Defaults.NativeLibraryExtension}";`
`23`	`23`	`}`
	`24`	`+`
	`25`	`+ partial class Urls`
	`26`	`+ {`
	`27`	`+ public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.sh");`
	`28`	`+ }`
`24`	`29`	`}`
`25`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,10 @@ partial class Paths`
`19`	`19`	`public static readonly string MonoRuntimeHostMingwNativeLibraryPrefix = Path.Combine ("..", "bin");`
`20`	`20`	`public const string NdkToolchainOSTag = "windows-x86_64";`
`21`	`21`	`}`
	`22`	`+`
	`23`	`+ partial class Urls`
	`24`	`+ {`
	`25`	`+ public static readonly Uri DotNetInstallScript = new Uri ("https://dot.net/v1/dotnet-install.ps1");`
	`26`	`+ }`
`22`	`27`	`}`
`23`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,7 @@ protected override void AddSteps (Context context)`
`21`	`21`	`// Install essential tools, should they be missing`
`22`	`22`	`context.SetCondition (KnownConditions.EnsureEssential, true);`
`23`	`23`
	`24`	`+ Steps.Add (new Step_InstallDotNetSDKs ());`
`24`	`25`	`Steps.Add (new Step_GenerateFiles (atBuildStart: true, onlyRequired: true));`
`25`	`26`	`Steps.Add (new Step_PrepareExternalJavaInterop ());`
`26`	`27`	`}`