Skip to content

[xaprepare] Download and install dotnet ourselves, with caching #7026

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 25, 2022
Merged

[xaprepare] Download and install dotnet ourselves, with caching #7026

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9ee8a15
[xaprepare] Download and install dotnet ourselves, with caching
grendello May 23, 2022
61255f5
Use different Split overload, to make it work on Windows
grendello May 23, 2022
06e0f0c
We need to support .NET 4.6.2
grendello May 23, 2022
010d46b
Remove unnecessary `--dry-run`
grendello May 24, 2022
9518175
Make size check failures non-critical
grendello May 24, 2022
4ebdf48
Make size checks failures critical, after all
grendello May 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 158 additions & 40 deletions build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ protected override async Task<bool> Execute (Context context)
dotnetPath = dotnetPath.TrimEnd (new char [] { Path.DirectorySeparatorChar });
var dotnetTool = Path.Combine (dotnetPath, "dotnet");

// Always delete the bin/$(Configuration)/dotnet/ directory
Utilities.DeleteDirectory (dotnetPath);

if (!await InstallDotNetAsync (context, dotnetPath, BuildToolVersion)) {
Log.ErrorLine ($"Installation of dotnet SDK {BuildToolVersion} failed.");
return false;
Expand Down Expand Up @@ -65,68 +62,189 @@ protected override async Task<bool> Execute (Context context)
return true;
}

async Task<bool> InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false)
async Task<bool> DownloadDotNetInstallScript (Context context, string dotnetScriptPath, Uri dotnetScriptUrl)
{
if (Directory.Exists (Path.Combine (dotnetPath, "sdk", version)) && !runtimeOnly) {
Log.Status ($"dotnet SDK version ");
Log.Status (version, ConsoleColor.Yellow);
Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "sdk", version), tailColor: ConsoleColor.Cyan);
return true;
string tempDotnetScriptPath = dotnetScriptPath + "-tmp";
Utilities.DeleteFile (tempDotnetScriptPath);

Log.StatusLine ("Downloading dotnet-install...");

(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl);
if (!success) {
string message;
if (status == HttpStatusCode.NotFound) {
message = "dotnet-install URL not found";
} else {
message = $"Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})";
}

return ReportAndCheckCached (message, quietOnError: true);
}

if (Directory.Exists (Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version)) && runtimeOnly) {
Log.Status ($"dotnet runtime version ");
Log.Status (version, ConsoleColor.Yellow);
Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version), tailColor: ConsoleColor.Cyan);
return true;
DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White);
await Download (context, dotnetScriptUrl, tempDotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus);

if (!File.Exists (tempDotnetScriptPath)) {
return ReportAndCheckCached ($"Download of dotnet-install from {dotnetScriptUrl} failed");
}

Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript;
string dotnetScriptPath = Path.Combine (dotnetPath, Path.GetFileName (dotnetScriptUrl.LocalPath));
if (File.Exists (dotnetScriptPath))
Utilities.DeleteFile (dotnetScriptPath);
Utilities.CopyFile (tempDotnetScriptPath, dotnetScriptPath);
Utilities.DeleteFile (tempDotnetScriptPath);
return true;

Log.StatusLine ("Downloading dotnet-install...");
bool ReportAndCheckCached (string message, bool quietOnError = false)
{
if (File.Exists (dotnetScriptPath)) {
Log.WarningLine (message);
Log.WarningLine ($"Using cached installation script found in {dotnetScriptPath}");
return true;
}

(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl);
if (!quietOnError) {
Log.ErrorLine (message);
Log.ErrorLine ($"Cached installation script not found in {dotnetScriptPath}");
}
return false;
}
}

async Task<bool> DownloadDotNetArchive (Context context, string archiveDestinationPath, Uri archiveUrl)
{
Log.StatusLine ("Downloading dotnet archive...");

(bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (archiveUrl);
if (!success) {
if (status == HttpStatusCode.NotFound)
Log.ErrorLine ("dotnet-install URL not found");
else
Log.ErrorLine ("Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})");
if (status == HttpStatusCode.NotFound) {
Log.ErrorLine ($"dotnet archive URL {archiveUrl} not found");
return false;
} else {
Log.WarningLine ($"Failed to obtain dotnet archive size. HTTP status code: {status} ({(int)status})");
}

return false;
}

string tempArchiveDestinationPath = archiveDestinationPath + "-tmp";
Utilities.DeleteFile (tempArchiveDestinationPath);

DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession);
Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White);
await Download (context, dotnetScriptUrl, dotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus);
Log.StatusLine ($" {context.Characters.Link} {archiveUrl}", ConsoleColor.White);
await Download (context, archiveUrl, tempArchiveDestinationPath, "dotnet archive", Path.GetFileName (archiveUrl.LocalPath), downloadStatus);

if (!File.Exists (dotnetScriptPath)) {
Log.ErrorLine ($"Download of dotnet-install from {dotnetScriptUrl} failed");
if (!File.Exists (tempArchiveDestinationPath)) {
return false;
}

var type = runtimeOnly ? "runtime" : "SDK";
Log.StatusLine ($"Installing dotnet {type} '{version}'...");
Utilities.CopyFile (tempArchiveDestinationPath, archiveDestinationPath);
Utilities.DeleteFile (tempArchiveDestinationPath);

return true;
}

string[] GetInstallationScriptArgs (string version, string dotnetPath, string dotnetScriptPath, bool onlyGetUrls, bool runtimeOnly)
{
List<string> args;
if (Context.IsWindows) {
var args = new List<string> {
args = new List<string> {
"-NoProfile", "-ExecutionPolicy", "unrestricted", "-file", dotnetScriptPath,
"-Version", version, "-InstallDir", dotnetPath, "-Verbose"
};
if (runtimeOnly)
if (runtimeOnly) {
args.AddRange (new string [] { "-Runtime", "dotnet" });
}
if (onlyGetUrls) {
args.Add ("-DryRun");
}

return Utilities.RunCommand ("powershell.exe", args.ToArray ());
} else {
var args = new List<string> {
dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose"
};
if (runtimeOnly)
args.AddRange (new string [] { "-Runtime", "dotnet" });
return args.ToArray ();
}

args = new List<string> {
dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose"
};

if (runtimeOnly) {
args.AddRange (new string [] { "-Runtime", "dotnet" });
}
if (onlyGetUrls) {
args.Add ("--dry-run");
Copy link
Contributor

@dellis1972 dellis1972 May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't we already add --dry-run on line 163? Do we need this as well or do we need the arg on line 163?

Copy link
Contributor Author

@grendello grendello May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good eye, it's a mistake, will fix

}

return args.ToArray ();
}

async Task<bool> InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false)
{
string cacheDir = context.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory);

return Utilities.RunCommand ("bash", args.ToArray ());
// Always delete the bin/$(Configuration)/dotnet/ directory
Utilities.DeleteDirectory (dotnetPath);

Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript;
string scriptFileName = Path.GetFileName (dotnetScriptUrl.LocalPath);
string cachedDotnetScriptPath = Path.Combine (cacheDir, scriptFileName);
if (!await DownloadDotNetInstallScript (context, cachedDotnetScriptPath, dotnetScriptUrl)) {
return false;
}

string dotnetScriptPath = Path.Combine (dotnetPath, scriptFileName);
Utilities.CopyFile (cachedDotnetScriptPath, dotnetScriptPath);

var type = runtimeOnly ? "runtime" : "SDK";

Log.StatusLine ($"Discovering download URLs for dotnet {type} '{version}'...");
string scriptCommand = Context.IsWindows ? "powershell.exe" : "bash";
string[] scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: true, runtimeOnly: runtimeOnly);
string scriptReply = Utilities.GetStringFromStdout (scriptCommand, scriptArgs);
var archiveUrls = new List<string> ();

char[] fieldSplitChars = new char[] { ':' };
foreach (string l in scriptReply.Split (new char[] { '\n' })) {
string line = l.Trim ();

if (!line.StartsWith ("dotnet-install: URL #", StringComparison.OrdinalIgnoreCase)) {
continue;
}

string[] parts = line.Split (fieldSplitChars, 3);
if (parts.Length < 3) {
Log.WarningLine ($"dotnet-install URL line has unexpected number of parts. Expected 3, got {parts.Length}");
Log.WarningLine ($"Line: {line}");
continue;
}

archiveUrls.Add (parts[2].Trim ());
}

if (archiveUrls.Count == 0) {
Log.WarningLine ("No dotnet archive URLs discovered, attempting to run the installation script");
scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: false, runtimeOnly: runtimeOnly);
return Utilities.RunCommand (scriptCommand, scriptArgs);
}

string? archivePath = null;
foreach (string url in archiveUrls) {
var archiveUrl = new Uri (url);
string archiveDestinationPath = Path.Combine (cacheDir, Path.GetFileName (archiveUrl.LocalPath));

if (File.Exists (archiveDestinationPath)) {
archivePath = archiveDestinationPath;
break;
}

if (await DownloadDotNetArchive (context, archiveDestinationPath, archiveUrl)) {
archivePath = archiveDestinationPath;
break;
}
}

if (String.IsNullOrEmpty (archivePath)) {
return false;
}

Log.StatusLine ($"Installing dotnet {type} '{version}'...");
return await Utilities.Unpack (archivePath, dotnetPath);
}

bool TestDotNetSdk (string dotnetTool)
Expand Down