Merge remote-tracking branch 'origin/dev/rolf/tests-windows-diagnose'…

… into net10.0-diagnose-windows-test-failures

.gitignore

@@ -1,3 +1,4 @@
+config.runsettings
 Build.props
 Make.config.inc
 Make.config.local

Make.config

@@ -313,10 +313,10 @@ MIN_TVOS_SIMULATOR_VERSION=15.0
 EXTRA_SIMULATORS=com.apple.pkg.iPhoneSimulatorSDK15_0 com.apple.pkg.AppleTVSimulatorSDK15_0 com.apple.pkg.WatchSimulatorSDK8_0
 
 INCLUDE_IOS=1
-INCLUDE_MAC=1
-INCLUDE_WATCH=1
-INCLUDE_TVOS=1
-INCLUDE_MACCATALYST=1
+INCLUDE_MAC=
+INCLUDE_WATCH=
+INCLUDE_TVOS=
+INCLUDE_MACCATALYST=
 INCLUDE_DEVICE=1
 INCLUDE_DOTNET_WATCHOS=
 INCLUDE_XAMARIN_LEGACY=

dotnet/targets/Xamarin.Shared.Sdk.targets

@@ -869,8 +869,6 @@
 			Condition="'$(IsMacEnabled)' == 'true'"
 			AdditionalArguments="$(_DittoArchitectures)"
 			CopyFromWindows="true"
-			ToolExe="$(DittoExe)"
-			ToolPath="$(DittoPath)"
 			Source="%(_DirectoriesToPublish.SourceDirectory)"
 			Destination="%(_DirectoriesToPublish.TargetDirectory)"
 			TouchDestinationFiles="true"

msbuild/Directory.Build.props

@@ -10,8 +10,12 @@
 			interesting ways. So in order to get the exact version, they're
 			enclosed in brackets.
 
+			Lists of versions can be found here:
+
+			https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/
+
 		-->
-		<MessagingVersion>[2.1.15]</MessagingVersion>
+		<MessagingVersion Condition="'$(MessagingVersion)' == ''">[2.1.59-pushpackage-diagnose-cancelled-posts]</MessagingVersion>
 		<HotRestartVersion>[1.1.7]</HotRestartVersion>
 	</PropertyGroup>
 	<Import Project="$(MSBuildThisFileDirectory)../Directory.Build.props" />

msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Diagnostics;
 using System.IO;
 using System.Threading.Tasks;
@@ -25,6 +26,10 @@ public ExecuteTaskMessageHandler ()
 
 		protected override async Task<ExecuteTaskResult> ExecuteAsync (ExecuteTaskMessage message)
 		{
+			var msg = $"{System.DateTime.UtcNow.ToString ("o")} ExecuteTaskMessageHandler.ExecuteAsync ({message.TaskName})\n{System.Environment.StackTrace}";
+			Console.WriteLine (msg);
+			Console.Error.WriteLine (msg);
+			tracer.Info (msg);
 			return await Task.Run (() => {
 				// We need to lock in order to change the current directory
 				lock (lockObject) {

msbuild/Xamarin.MacDev.Tasks/Decompress.cs

@@ -4,6 +4,7 @@
 using System.IO;
 using System.IO.Compression;
 using System.Reflection;
+using System.Threading;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
@@ -81,9 +82,10 @@ public static bool IsCompressed (string path)
 		/// <param name="zip">The zip to search in</param>
 		/// <param name="resource">The relative path inside the zip to extract (may be a file or a directory).</param>
 		/// <param name="decompressionDir">The location on disk to store the extracted results</param>
+		/// <param name="cancellationToken">The cancellation token (if any=</param>
 		/// <param name="decompressedResource">The location on disk to the extracted resource</param>
 		/// <returns></returns>
-		public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List<string> createdFiles, [NotNullWhen (true)] out string? decompressedResource)
+		public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List<string> createdFiles, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? decompressedResource)
 		{
 			decompressedResource = Path.Combine (decompressionDir, resource);
 
@@ -101,11 +103,11 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso
 
 			bool rv;
 			if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
-				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken);
 			} else if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("XAMARIN_USE_SYSTEM_IO_COMPRESSION"))) {
-				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken);
 			} else {
-				rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir, cancellationToken);
 			}
 
 			if (rv) {
@@ -128,7 +130,7 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso
 		// The dir separator character in zip files is always "/", even on Windows
 		const char zipDirectorySeparator = '/';
 
-		static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir)
+		static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken)
 		{
 			Directory.CreateDirectory (decompressionDir);
 			var args = new List<string> {
@@ -157,11 +159,11 @@ static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string r
 				args.Add (zipPattern);
 			}
 
-			var rv = XamarinTask.ExecuteAsync (log, "unzip", args).Result;
+			var rv = XamarinTask.ExecuteAsync (log, "unzip", args, cancellationToken: cancellationToken).Result;
 			return rv.ExitCode == 0;
 		}
 
-		static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir)
+		static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken)
 		{
 			var rv = true;
 
@@ -172,6 +174,7 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 
 			using var archive = ZipFile.OpenRead (zip);
 			foreach (var entry in archive.Entries) {
+				cancellationToken?.ThrowIfCancellationRequested ();
 				var entryPath = entry.FullName;
 				if (entryPath.Length == 0)
 					continue;
@@ -207,7 +210,11 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 					Directory.CreateDirectory (Path.GetDirectoryName (targetPath));
 					using var streamWrite = File.OpenWrite (targetPath);
 					using var streamRead = entry.Open ();
-					streamRead.CopyTo (streamWrite);
+#if NET
+					streamRead.CopyToAsync (streamWrite, cancellationToken ?? CancellationToken.None).Wait ();
+#else
+					streamRead.CopyToAsync (streamWrite, 81920 /* default buffer size according to docs */, cancellationToken ?? CancellationToken.None).Wait ();
+#endif
 					log.LogMessage (MessageImportance.Low, "Extracted {0} into {1}", entryPath, targetPath);
 				}
 			}

msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs

@@ -13,16 +13,23 @@ public class Copy : Microsoft_Build_Tasks_Core::Microsoft.Build.Tasks.Copy {
 		public string SessionId { get; set; } = string.Empty;
 		public override bool Execute ()
 		{
-			if (!this.ShouldExecuteRemotely (SessionId))
-				return base.Execute ();
+			try {
+				if (!this.ShouldExecuteRemotely (SessionId))
+					return base.Execute ();
 
-			var taskRunner = new TaskRunner (SessionId, BuildEngine4);
+				var taskRunner = new TaskRunner (SessionId, BuildEngine4);
 
-			if (SourceFiles?.Any () == true) {
-				taskRunner.FixReferencedItems (this, SourceFiles);
-			}
+				if (SourceFiles?.Any () == true) {
+					taskRunner.FixReferencedItems (this, SourceFiles);
+				}
 
-			return taskRunner.RunAsync (this).Result;
+				return taskRunner.RunAsync (this).Result;
+			} catch (Exception ex) {
+				Log.LogError ($"Copy failed due to exception: {ex.Message}");
+				Log.LogError ($"Stack trace:\n{ex.StackTrace}");
+				Log.LogErrorFromException (ex);
+				return false;
+			}
 		}
 	}
 }

msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs

@@ -2,16 +2,19 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Threading;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
 
 using Xamarin.Messaging.Build.Client;
+using Xamarin.Utils;
 
 namespace Xamarin.MacDev.Tasks {
-	public class Ditto : XamarinToolTask, ITaskCallback {
+	public class Ditto : XamarinTask, ITaskCallback, ICancelableTask {
 		#region Inputs
 
 		public string? AdditionalArguments { get; set; }
@@ -40,44 +43,53 @@ public class Ditto : XamarinToolTask, ITaskCallback {
 
 		#endregion
 
-		protected override string ToolName {
-			get { return "ditto"; }
-		}
-
-		protected override string GenerateFullPathToTool ()
-		{
-			if (!string.IsNullOrEmpty (ToolPath))
-				return Path.Combine (ToolPath, ToolExe);
-
-			var path = Path.Combine ("/usr/bin", ToolExe);
-
-			return File.Exists (path) ? path : ToolExe;
-		}
-
-		protected override string GenerateCommandLineCommands ()
-		{
-			var args = new CommandLineArgumentBuilder ();
-
-			args.AddQuoted (Path.GetFullPath (Source!.ItemSpec));
-			args.AddQuoted (Path.GetFullPath (Destination!.ItemSpec));
-			if (!string.IsNullOrEmpty (AdditionalArguments))
-				args.Add (AdditionalArguments);
-
-			return args.ToString ();
-		}
+		CancellationTokenSource? cancellationTokenSource;
 
+		static int counter;
 		public override bool Execute ()
 		{
+			var id = Interlocked.Increment (ref counter);
+
 			if (ShouldExecuteRemotely ()) {
+				LogLine ($"{id} Ditto.Execute () will execute remotely");
 				var taskRunner = new TaskRunner (SessionId, BuildEngine4);
 
 				taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! });
 
-				return taskRunner.RunAsync (this).Result;
+				LogLine ($"{id} Ditto.Execute () about to execute remotely");
+				var rv = taskRunner.RunAsync (this).Result;
+				LogLine ($"{id} Ditto.Execute () executed remotely: {rv}");
+				return rv;
 			}
 
-			if (!base.Execute ())
+			var src = Source!.ItemSpec;
+			if (!File.Exists (src) && !Directory.Exists (src)) {
+				Log.LogError ($"The source {src} does not exist.");
 				return false;
+			}
+
+			var args = new List<string> ();
+			args.Add (Path.GetFullPath (Source!.ItemSpec));
+			args.Add (Path.GetFullPath (Destination!.ItemSpec));
+#if NET
+			if (!string.IsNullOrEmpty (AdditionalArguments)) {
+#else
+			if (AdditionalArguments is not null && !string.IsNullOrEmpty (AdditionalArguments)) {
+#endif
+				if (StringUtils.TryParseArguments (AdditionalArguments, out var additionalArgs, out var ex)) {
+					args.AddRange (additionalArgs);
+				} else {
+					Log.LogError ("Unable to parse the AdditionalArguments: {0}", AdditionalArguments);
+					return false;
+				}
+			}
+
+			LogLine ($"{id} Ditto.Execute () about to execute locally. Args: {string.Join (" ", args)}\n{Environment.StackTrace}");
+			var watch = Stopwatch.StartNew ();
+			cancellationTokenSource = new CancellationTokenSource ();
+			cancellationTokenSource.CancelAfter (TimeSpan.FromSeconds (30)); // FIXME: remove this
+			ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait ();
+			LogLine ($"{id} Ditto.Execute () executed locally in {watch.Elapsed.TotalSeconds} seconds. Args: {string.Join (" ", args)}");
 
 			// Create a list of all the files we've copied
 			var copiedFiles = new List<ITaskItem> ();
@@ -96,18 +108,13 @@ public override bool Execute ()
 			return !Log.HasLoggedErrors;
 		}
 
-		protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
-		{
-			// TODO: do proper parsing of error messages and such
-			Log.LogMessage (messageImportance, "{0}", singleLine);
-		}
-
-		public override void Cancel ()
+		public void Cancel ()
 		{
-			base.Cancel ();
-
-			if (ShouldExecuteRemotely ())
+			if (ShouldExecuteRemotely ()) {
 				BuildConnection.CancelAsync (BuildEngine4).Wait ();
+			} else {
+				cancellationTokenSource?.Cancel ();
+			}
 		}
 
 		public IEnumerable<ITaskItem> GetAdditionalItemsToBeCopied ()