From 8894f7700022814bec464ce2c7e659a04c7ba090 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 25 Apr 2024 16:44:02 +0200 Subject: [PATCH 01/59] [DRAFT] PR to diagnose remote mac testing problems. --- msbuild/Directory.Build.props | 2 +- .../Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 572b64c1b415..90363b347cee 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -11,7 +11,7 @@ enclosed in brackets. --> - [1.14.6] + [2.0.42-diagnose-cancelled-posts] [1.1.7] diff --git a/msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs b/msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs index 3679d861f69e..fa3515a4eea8 100644 --- a/msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs +++ b/msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs @@ -13,16 +13,23 @@ public class Copy : Microsoft_Build_Tasks_Core::Microsoft.Build.Tasks.Copy { public string SessionId { get; set; } = string.Empty; public override bool Execute () { - if (!this.ShouldExecuteRemotely (SessionId)) - return base.Execute (); + try { + if (!this.ShouldExecuteRemotely (SessionId)) + return base.Execute (); - var taskRunner = new TaskRunner (SessionId, BuildEngine4); + var taskRunner = new TaskRunner (SessionId, BuildEngine4); - if (SourceFiles?.Any () == true) { - taskRunner.FixReferencedItems (this, SourceFiles); - } + if (SourceFiles?.Any () == true) { + taskRunner.FixReferencedItems (this, SourceFiles); + } - return taskRunner.RunAsync (this).Result; + return taskRunner.RunAsync (this).Result; + } catch (Exception ex) { + Log.LogError ($"Copy failed due to exception: {ex.Message}"); + Log.LogError ($"Stack trace:\n{ex.StackTrace}"); + Log.LogErrorFromException (ex); + return false; + } } } } From 38c3d472e4acabab4e992d045c998d5946357f48 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 10 Sep 2024 15:41:18 +0200 Subject: [PATCH 02/59] Bump --- msbuild/Directory.Build.props | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 90363b347cee..c52b0afcafad 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -10,8 +10,12 @@ interesting ways. So in order to get the exact version, they're enclosed in brackets. + Lists of versions can be found here: + + https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ + --> - [2.0.42-diagnose-cancelled-posts] + [2.1.30-diagnose-cancelled-posts] [1.1.7] From 78116d7852539ac3837eb39a5c8d3e653391c80c Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 11 Sep 2024 18:59:26 +0200 Subject: [PATCH 03/59] bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index c52b0afcafad..a8703faf0421 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.30-diagnose-cancelled-posts] + [2.1.31-diagnose-cancelled-posts] [1.1.7] From 8f268a3c464e8b2a6852ee46ec3e795895d4a31e Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 16 Sep 2024 20:45:52 +0200 Subject: [PATCH 04/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index a8703faf0421..00aa228ae821 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.31-diagnose-cancelled-posts] + [2.1.32-diagnose-cancelled-posts] [1.1.7] From 1c604f66e1925a9cd92b7a366798015a1a66c64a Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 18 Sep 2024 10:35:23 +0200 Subject: [PATCH 05/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 00aa228ae821..68f71bdb2c5a 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.32-diagnose-cancelled-posts] + [2.1.33-diagnose-cancelled-posts] [1.1.7] From b1d60dd6fcf2075e501bba5873e5a9f725ee782a Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Thu, 12 Sep 2024 03:33:34 -0700 Subject: [PATCH 06/59] Disable/reenable agents using dynamically obtained access token (#21216) Replace `MacPoolAccessToken` with a dynamically obtained Azure DevOps access token. The `MacPoolAccessToken` is backed by the `botdeploy--azdo--token--register--untrusted` secret, which is limited to a 7-day lifespan. This change removes the dependency on that token --- .../devops/automation/templates/variables.yml | 2 -- .../templates/windows/generate-token.yml | 25 +++++++++++++++++++ .../templates/windows/reenable-mac.yml | 10 +++++++- .../templates/windows/reserve-mac.yml | 9 ++++++- 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 tools/devops/automation/templates/windows/generate-token.yml diff --git a/tools/devops/automation/templates/variables.yml b/tools/devops/automation/templates/variables.yml index 38e6ddc50d75..cabd95d39e32 100644 --- a/tools/devops/automation/templates/variables.yml +++ b/tools/devops/automation/templates/variables.yml @@ -36,5 +36,3 @@ variables: value: true - name: BUILD_REVISION value: azure-devops-$(Build.SourceVersion) -- name: MacPoolAccessToken - value: $(botdeploy--azdo--token--register--untrusted) diff --git a/tools/devops/automation/templates/windows/generate-token.yml b/tools/devops/automation/templates/windows/generate-token.yml new file mode 100644 index 000000000000..97001b181403 --- /dev/null +++ b/tools/devops/automation/templates/windows/generate-token.yml @@ -0,0 +1,25 @@ +# Sets the AzDO.BearerToken variable that can be used as the auth token to disable/reenable agents +steps: +- template: azure-tools/az-client-update.yml@yaml-templates # AzureCLI step below requires that AzClient 2.x is installed on the agent + parameters: + platform: 'All' # Update Az tools for both Windows and Mac agents + version: '2.62.0' + +# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-security-configuration/configuration-guides/pat-burndown-guidance#authentication-from-pipelines +# Requires Azure client 2.x +- task: AzureCLI@2 + displayName: 'AzDO.BearerToken based on service connection' + enabled: true + inputs: + azureSubscription: 'DevDiv - SharedUntrustedAgentPool-Manage' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # if this fails, check out this bash script that includes diagnostics: + # https://gist.github.com/johnterickson/19f80a3e969e39f1000d118739176e62 + + # Note that the resource is specified to limit the token to Azure DevOps + $token = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv + + Write-Host "Setting AzDO.BearerToken" + Write-Host "##vso[task.setvariable variable=AzDO.BearerToken;issecret=true]${token}" diff --git a/tools/devops/automation/templates/windows/reenable-mac.yml b/tools/devops/automation/templates/windows/reenable-mac.yml index b06f211a807a..269802117011 100644 --- a/tools/devops/automation/templates/windows/reenable-mac.yml +++ b/tools/devops/automation/templates/windows/reenable-mac.yml @@ -47,9 +47,17 @@ steps: condition: always() continueOnError: true +# Sets the AzDO.BearerToken variable used as the auth token to disable/reenable agents +- template: ./generate-token.yml + - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 - $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken) + + $azdoBearerToken = "$(AzDO.BearerToken)" + $azdoBearerTokenHint = $azdoBearerToken.Substring(0, 8) + Write-Host "AzDO.BearerToken (hint): ${azdoBearerTokenHint}" + + $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $azdoBearerToken # get the pool and the agent objects and enable the bot $pool = $vsts.Pools.GetPool("$Env:MAC_AGENT_POOL") diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index a1fef1d8ec47..afbd6e7dc216 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -64,13 +64,20 @@ steps: AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_P12: ${{ parameters.xqaCertPass }} AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_2_P12: ${{ parameters.xqaCertPass }} +# Sets the AzDO.BearerToken variable used as the auth token to disable/reenable agents +- template: ./generate-token.yml + - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 New-Item -Path "$($Env:HOME)" -Name "remote_build_testing" -Force -ItemType "directory" New-Item -Path "$($Env:HOME)/remote_build_testing" -Name "BuildId.txt" -ItemType "file" -Force -Value "$($Env:BUILD_BUILDID)" - $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $(MacPoolAccessToken) + $azdoBearerToken = "$(AzDO.BearerToken)" + $azdoBearerTokenHint = $azdoBearerToken.Substring(0, 8) + Write-Host "AzDO.BearerToken (hint): ${azdoBearerTokenHint}" + + $vsts = New-VSTS -Org "devdiv" -Project "DevDiv" -Token $azdoBearerToken # get the pool and the agent objects and disable the bot $pool = $vsts.Pools.GetPool("${{ parameters.macPool }}") From aef7387d995695855b17b17fa48ff655214829fb Mon Sep 17 00:00:00 2001 From: Emanuel Fernandez Dell'Oca Date: Mon, 24 Jun 2024 08:10:15 -0400 Subject: [PATCH 07/59] Migrate Messaging Build agent to .NET 8 This is part of the effort to migrate the Pair to Mac agents .NET. As the Xamarin.iOS.Tasks.Windows project targets netstandard2.0, I'm removing the Build agent reference, and modifying the Makefile to take it from it's output directory. Note: the agent zip file is generated in the intermediate output directory. --------- Co-authored-by: Alex Soto Co-authored-by: Mauro Agnoletti Co-authored-by: Rolf Bjarne Kvinge --- msbuild/ILMerge.targets | 1 + msbuild/Makefile | 2 +- .../Xamarin.Messaging.Build/BuildAgent.cs | 3 -- .../Handlers/CopyItemMessageHandler.cs | 9 ++-- .../Xamarin.Messaging.Build.csproj | 3 +- .../MSBStrings.resx | 13 ------ .../Tasks/VerifyXcodeVersion.cs | 46 ------------------- msbuild/Xamarin.Shared/Xamarin.Shared.targets | 1 - .../Xamarin.Messaging.Apple.targets | 7 --- .../Xamarin.iOS.Tasks.Windows.csproj | 5 -- 10 files changed, 10 insertions(+), 80 deletions(-) delete mode 100644 msbuild/Xamarin.MacDev.Tasks/Tasks/VerifyXcodeVersion.cs diff --git a/msbuild/ILMerge.targets b/msbuild/ILMerge.targets index 27c7e82b8338..f9720e901ebc 100644 --- a/msbuild/ILMerge.targets +++ b/msbuild/ILMerge.targets @@ -42,6 +42,7 @@ + diff --git a/msbuild/Makefile b/msbuild/Makefile index 44d9aef1da8c..373a0c5642dc 100644 --- a/msbuild/Makefile +++ b/msbuild/Makefile @@ -491,9 +491,9 @@ DOTNET_IOS_WINDOWS_OUTPUT_FILES = \ iSign.Core.pdb \ Xamarin.iOS.Windows.Client.pdb \ Broker.zip \ - Build.zip \ DOTNET_IOS_WINDOWS_FILES = $(IOS_WINDOWS_TARGETS) $(foreach file,$(DOTNET_IOS_WINDOWS_OUTPUT_FILES),Xamarin.iOS.Tasks.Windows/bin/$(CONFIG)/$(TARGETFRAMEWORK)/$(WINDOWSRUNTIMEIDENTIFIER)/$(file)) +DOTNET_IOS_WINDOWS_FILES += Messaging/Xamarin.Messaging.Build/obj/$(CONFIG)/Build.zip DOTNET_IOS_WINDOWS_MOBILEDEVICE_TOOLS_X86 = $(foreach file,$(IOS_WINDOWS_MOBILEDEVICE_TOOLS),Xamarin.iOS.Tasks.Windows/bin/$(CONFIG)/$(TARGETFRAMEWORK)/$(WINDOWSRUNTIMEIDENTIFIER)/imobiledevice-x86/$(file).*) DOTNET_IOS_WINDOWS_MOBILEDEVICE_TOOLS_X64 = $(foreach file,$(IOS_WINDOWS_MOBILEDEVICE_TOOLS),Xamarin.iOS.Tasks.Windows/bin/$(CONFIG)/$(TARGETFRAMEWORK)/$(WINDOWSRUNTIMEIDENTIFIER)/imobiledevice-x64/$(file).*) diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/BuildAgent.cs b/msbuild/Messaging/Xamarin.Messaging.Build/BuildAgent.cs index b327d33eaba7..658832101298 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/BuildAgent.cs +++ b/msbuild/Messaging/Xamarin.Messaging.Build/BuildAgent.cs @@ -49,9 +49,6 @@ await TryRegisterHandlerAsync (new CopyItemMessageHandler ()) .ConfigureAwait (continueOnCapturedContext: false); await TryRegisterHandlerAsync (new GetItemMessageHandler ()) .ConfigureAwait (continueOnCapturedContext: false); - - await TryRegisterHandlerAsync (new VerifyXcodeVersionMessageHandler ()) - .ConfigureAwait (continueOnCapturedContext: false); await TryRegisterHandlerAsync (new GetUniversalTargetIdentifierMessageHandler ()) .ConfigureAwait (continueOnCapturedContext: false); } diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/CopyItemMessageHandler.cs b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/CopyItemMessageHandler.cs index 89ecd1fefd0e..2e6e4c4d8391 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/CopyItemMessageHandler.cs +++ b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/CopyItemMessageHandler.cs @@ -17,10 +17,13 @@ await Task.Run (async () => { message.SessionId, PlatformPath.GetPathForCurrentPlatform (message.ItemSpec))); - if (File.Exists (targetPath)) + if (File.Exists (targetPath)) { File.Delete (targetPath); - else if (!Directory.Exists (Path.GetDirectoryName (targetPath))) - Directory.CreateDirectory (Path.GetDirectoryName (targetPath)); + } else { + var directoryName = Path.GetDirectoryName (targetPath); + if (directoryName is not null && !Directory.Exists (directoryName)) + Directory.CreateDirectory (directoryName); + } using (var file = File.Create (targetPath)) using (var stream = new MemoryStream (message.Content)) diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj b/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj index 4700701bfdce..4845fe73179a 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj +++ b/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj @@ -1,6 +1,6 @@ - net472 + net8.0 Exe Build $(NoWarn);NU1603 @@ -10,6 +10,7 @@ enable latest Nullable + false diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx index 4c6fb887fd07..4d591d32a395 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx @@ -876,19 +876,6 @@ Could not resolve UTI for {0} in catalog {1} - - An error occurred while trying to verify the compatibility between Xcode and the .NET SDK - - - - The project was built with a version of Xcode ({0}) that is not compatible which may result in unexpected warnings or errors. Please install Xcode '{1}' or visit the documentation to learn more. - - - - An error occurred while trying to verify the compatibility between Xcode and the .NET SDK. Details: {0} - - - The provided Archive Path is not from a valid archive file (.xcarchive). Archive Path: '{0}' diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/VerifyXcodeVersion.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/VerifyXcodeVersion.cs deleted file mode 100644 index 1a283be0b827..000000000000 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/VerifyXcodeVersion.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Xamarin.Localization.MSBuild; -using Xamarin.MacDev.Tasks; -using Xamarin.Messaging.Build.Client; -using Xamarin.Messaging.Build.Contracts; -using Threading = System.Threading.Tasks; - -namespace Xamarin.MacDev.Tasks { - public class VerifyXcodeVersion : XamarinTask { - public override bool Execute () => ExecuteAsync ().Result; - - async Threading.Task ExecuteAsync () - { - try { - var connection = await BuildConnection.GetAsync (BuildEngine4).ConfigureAwait (continueOnCapturedContext: false); - var buildClient = connection.GetClient (SessionId) as BuildClient; - - if (!connection.IsConnected || buildClient is null) { - Log.LogWarning (MSBStrings.E0179, nameof (VerifyXcodeVersion)); - - return true; - } - - var xcodeVerificationResult = await buildClient - .RunMessagingAsync (new VerifyXcodeVersionMessage ()) - .ConfigureAwait (continueOnCapturedContext: false); - - if (!xcodeVerificationResult.IsCompatibleVersion) { - if (string.IsNullOrEmpty (xcodeVerificationResult.XcodeVersion) || string.IsNullOrEmpty (xcodeVerificationResult.RecommendedXcodeVersion)) { - Log.LogWarning (MSBStrings.E0183); - } else { - Log.LogWarning (MSBStrings.E0184, xcodeVerificationResult.XcodeVersion, xcodeVerificationResult.RecommendedXcodeVersion); - } - } - - return true; - } catch (Exception ex) { - Log.LogWarning (MSBStrings.E0185, ex.Message); - - return true; - } - } - } -} diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.targets index ad9714ec9a33..04ee48f16549 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.targets @@ -105,7 +105,6 @@ Copyright (C) 2018 Microsoft. All rights reserved. - diff --git a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.Messaging.Apple.targets b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.Messaging.Apple.targets index c5c217cd02a8..b2f8693b7813 100644 --- a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.Messaging.Apple.targets +++ b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.Messaging.Apple.targets @@ -23,13 +23,6 @@ False - - - - - - - True diff --git a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Tasks.Windows.csproj b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Tasks.Windows.csproj index 5116271a82b7..1173aaeb0905 100644 --- a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Tasks.Windows.csproj +++ b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Tasks.Windows.csproj @@ -29,11 +29,6 @@ - - all - False - True - From d08dc2b2e0fdc1245c408e5fb0a2b2408dbebee9 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 19 Sep 2024 11:39:41 +0200 Subject: [PATCH 08/59] [msbuild] Allow the remote Build process to run on newer .NET versions than the one it was built for. --- .../Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj b/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj index 4845fe73179a..8a3428ab3070 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj +++ b/msbuild/Messaging/Xamarin.Messaging.Build/Xamarin.Messaging.Build.csproj @@ -11,6 +11,7 @@ latest Nullable false + LatestMajor From bae6abe2fda40c07a112a5974a111db55dd8baef Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 19 Sep 2024 16:51:04 +0200 Subject: [PATCH 09/59] [tests] Improve failure message when a process times out. --- tests/common/DotNet.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/common/DotNet.cs b/tests/common/DotNet.cs index 15be67b95b77..b058cd1756cb 100644 --- a/tests/common/DotNet.cs +++ b/tests/common/DotNet.cs @@ -270,7 +270,11 @@ public static ExecutionResult Execute (string verb, string project, Dictionary Date: Thu, 19 Sep 2024 16:51:15 +0200 Subject: [PATCH 10/59] [tests] Bump timeout to 30 min. --- tests/dotnet/UnitTests/WindowsTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dotnet/UnitTests/WindowsTest.cs b/tests/dotnet/UnitTests/WindowsTest.cs index af089615c563..3d5b7870b90d 100644 --- a/tests/dotnet/UnitTests/WindowsTest.cs +++ b/tests/dotnet/UnitTests/WindowsTest.cs @@ -187,7 +187,7 @@ public void BundleStructureWithRemoteMac (ApplePlatform platform, string runtime // Copy the app bundle to Windows so that we can inspect the results. properties ["CopyAppBundleToWindows"] = "true"; - var rv = DotNet.AssertBuild (project_path, properties); + var rv = DotNet.AssertBuild (project_path, properties, timeout: TimeSpan.FromMinutes (30)); var warnings = BinLog.GetBuildLogWarnings (rv.BinLogPath).ToArray (); var warningMessages = BundleStructureTest.FilterWarnings (warnings, canonicalizePaths: true); From f8b3cc11cf82303843ed65de73bbb99589955cd5 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 20 Sep 2024 12:52:21 +0200 Subject: [PATCH 11/59] Verbose --- tests/common/DotNet.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/common/DotNet.cs b/tests/common/DotNet.cs index b058cd1756cb..2825f316d99f 100644 --- a/tests/common/DotNet.cs +++ b/tests/common/DotNet.cs @@ -244,7 +244,8 @@ public static ExecutionResult Execute (string verb, string project, Dictionary Date: Fri, 20 Sep 2024 20:00:34 +0200 Subject: [PATCH 12/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 68f71bdb2c5a..66ce89910693 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.33-diagnose-cancelled-posts] + [2.1.35-diagnose-cancelled-posts] [1.1.7] From 4cfd9d0c98ca69b0cf621a7784cb3ed679feaae3 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 23 Sep 2024 20:59:21 +0200 Subject: [PATCH 13/59] BUmp --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 66ce89910693..19a91ff4c472 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.35-diagnose-cancelled-posts] + [2.1.36-diagnose-cancelled-posts] [1.1.7] From 9ef843e372152aba4bfca43e932cfb3de2e56209 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 20 Sep 2024 19:04:38 +0200 Subject: [PATCH 14/59] [msbuild] Convert Ditto to a XamarinTask. --- dotnet/targets/Xamarin.Shared.Sdk.targets | 2 - msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 66 ++++++++----------- .../Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs | 5 +- msbuild/Xamarin.Shared/Xamarin.Shared.targets | 2 - .../Xamarin.WatchOS.App.Common.targets | 4 -- .../Xamarin.Shared/Xamarin.iOS.Common.targets | 6 -- 6 files changed, 31 insertions(+), 54 deletions(-) diff --git a/dotnet/targets/Xamarin.Shared.Sdk.targets b/dotnet/targets/Xamarin.Shared.Sdk.targets index 293b547d7547..ba6938dd41dc 100644 --- a/dotnet/targets/Xamarin.Shared.Sdk.targets +++ b/dotnet/targets/Xamarin.Shared.Sdk.targets @@ -850,8 +850,6 @@ Condition="'$(IsMacEnabled)' == 'true'" AdditionalArguments="$(_DittoArchitectures)" CopyFromWindows="true" - ToolExe="$(DittoExe)" - ToolPath="$(DittoPath)" Source="%(_DirectoriesToPublish.SourceDirectory)" Destination="%(_DirectoriesToPublish.TargetDirectory)" TouchDestinationFiles="true" diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index a4a849dfd7df..e48179691a2b 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -4,14 +4,16 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Messaging.Build.Client; +using Xamarin.Utils; namespace Xamarin.MacDev.Tasks { - public class Ditto : XamarinToolTask, ITaskCallback { + public class Ditto : XamarinTask, ICancelableTask { #region Inputs public string? AdditionalArguments { get; set; } @@ -40,31 +42,7 @@ public class Ditto : XamarinToolTask, ITaskCallback { #endregion - protected override string ToolName { - get { return "ditto"; } - } - - protected override string GenerateFullPathToTool () - { - if (!string.IsNullOrEmpty (ToolPath)) - return Path.Combine (ToolPath, ToolExe); - - var path = Path.Combine ("/usr/bin", ToolExe); - - return File.Exists (path) ? path : ToolExe; - } - - protected override string GenerateCommandLineCommands () - { - var args = new CommandLineArgumentBuilder (); - - args.AddQuoted (Path.GetFullPath (Source!.ItemSpec)); - args.AddQuoted (Path.GetFullPath (Destination!.ItemSpec)); - if (!string.IsNullOrEmpty (AdditionalArguments)) - args.Add (AdditionalArguments); - - return args.ToString (); - } + CancellationTokenSource? cancellationTokenSource; public override bool Execute () { @@ -76,8 +54,25 @@ public override bool Execute () return taskRunner.RunAsync (this).Result; } - if (!base.Execute ()) - return false; + var args = new List (); + args.Add (Path.GetFullPath (Source!.ItemSpec)); + args.Add (Path.GetFullPath (Destination!.ItemSpec)); +#if NET + if (!string.IsNullOrEmpty (AdditionalArguments)) { +#else + if (AdditionalArguments is not null && !string.IsNullOrEmpty (AdditionalArguments)) { +#endif + if (StringUtils.TryParseArguments (AdditionalArguments, out var additionalArgs, out var ex)) { + args.AddRange (additionalArgs); + } else { + Log.LogError ("Unable to parse the AdditionalArguments: {0}", AdditionalArguments); + return false; + } + } + + cancellationTokenSource = new CancellationTokenSource (); + cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this + ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait (); // Create a list of all the files we've copied var copiedFiles = new List (); @@ -96,18 +91,13 @@ public override bool Execute () return !Log.HasLoggedErrors; } - protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance) - { - // TODO: do proper parsing of error messages and such - Log.LogMessage (messageImportance, "{0}", singleLine); - } - - public override void Cancel () + public void Cancel () { - base.Cancel (); - - if (ShouldExecuteRemotely ()) + if (ShouldExecuteRemotely ()) { BuildConnection.CancelAsync (BuildEngine4).Wait (); + } else { + cancellationTokenSource?.Cancel (); + } } public IEnumerable GetAdditionalItemsToBeCopied () diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs index e17775c17c75..ab5fc5e04ed0 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Tasks; @@ -120,7 +121,7 @@ protected System.Threading.Tasks.Task ExecuteAsync (string fileName, return ExecuteAsync (Log, fileName, arguments, sdkDevPath, environment, mergeOutput, showErrorIfFailure, workingDirectory); } - internal protected static async System.Threading.Tasks.Task ExecuteAsync (TaskLoggingHelper log, string fileName, IList arguments, string? sdkDevPath = null, Dictionary? environment = null, bool mergeOutput = true, bool showErrorIfFailure = true, string? workingDirectory = null) + internal protected static async System.Threading.Tasks.Task ExecuteAsync (TaskLoggingHelper log, string fileName, IList arguments, string? sdkDevPath = null, Dictionary? environment = null, bool mergeOutput = true, bool showErrorIfFailure = true, string? workingDirectory = null, CancellationToken? cancellationToken = null) { // Create a new dictionary if we're given one, to make sure we don't change the caller's dictionary. var launchEnvironment = environment is null ? new Dictionary () : new Dictionary (environment); @@ -128,7 +129,7 @@ internal protected static async System.Threading.Tasks.Task ExecuteAs launchEnvironment ["DEVELOPER_DIR"] = sdkDevPath; log.LogMessage (MessageImportance.Normal, MSBStrings.M0001, fileName, StringUtils.FormatArguments (arguments)); - var rv = await Execution.RunAsync (fileName, arguments, environment: launchEnvironment, mergeOutput: mergeOutput, workingDirectory: workingDirectory); + var rv = await Execution.RunAsync (fileName, arguments, environment: launchEnvironment, mergeOutput: mergeOutput, workingDirectory: workingDirectory, cancellationToken: cancellationToken); log.LogMessage (rv.ExitCode == 0 ? MessageImportance.Low : MessageImportance.High, MSBStrings.M0002, fileName, rv.ExitCode); // Show the output diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.targets index 1b5468798c80..ef2cf922f259 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.targets @@ -2510,8 +2510,6 @@ Copyright (C) 2018 Microsoft. All rights reserved. @@ -93,8 +91,6 @@ Copyright (C) 2015-2016 Xamarin. All rights reserved. SessionId="$(BuildSessionId)" Condition="'$(IsMacEnabled)'" AdditionalArguments="$(WKDittoArchitectures)" - ToolExe="$(DittoExe)" - ToolPath="$(DittoPath)" Source="$(_NativeWatchApp)" Destination="$(_AppBundlePath)_WatchKitStub\WK" /> diff --git a/msbuild/Xamarin.Shared/Xamarin.iOS.Common.targets b/msbuild/Xamarin.Shared/Xamarin.iOS.Common.targets index ffefb06341a4..98b7e72ebc98 100644 --- a/msbuild/Xamarin.Shared/Xamarin.iOS.Common.targets +++ b/msbuild/Xamarin.Shared/Xamarin.iOS.Common.targets @@ -371,8 +371,6 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved. @@ -391,8 +389,6 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved. @@ -481,8 +477,6 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved. From cae9bcba1162ee283f43b9c534943730193759e6 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 24 Sep 2024 17:39:35 +0200 Subject: [PATCH 15/59] Add debug spew. --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index e48179691a2b..b70c7bbafd5e 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -47,10 +47,12 @@ public class Ditto : XamarinTask, ICancelableTask { public override bool Execute () { if (ShouldExecuteRemotely ()) { + Log.LogWarning ($"Ditto.Execute () about to execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! }); + Log.LogWarning ($"Ditto.Execute () executed remotely"); return taskRunner.RunAsync (this).Result; } @@ -70,9 +72,11 @@ public override bool Execute () } } + Log.LogWarning ($"Ditto.Execute () about to execute locally"); cancellationTokenSource = new CancellationTokenSource (); cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait (); + Log.LogWarning ($"Ditto.Execute () executed locally"); // Create a list of all the files we've copied var copiedFiles = new List (); From d563a401bcc4b31f6cea2bb5e8ca3555107dc279 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 24 Sep 2024 08:43:21 +0200 Subject: [PATCH 16/59] [msbuild] Make unzip cancellable. --- msbuild/Xamarin.MacDev.Tasks/Decompress.cs | 23 +++++--- .../Tasks/ResolveNativeReferences.cs | 52 +++++++++++-------- msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs | 17 ++++-- .../ResolveNativeReferencesTaskTest.cs | 7 +-- 4 files changed, 63 insertions(+), 36 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Decompress.cs b/msbuild/Xamarin.MacDev.Tasks/Decompress.cs index ed47c46bf98d..4f2f8d5c0636 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Decompress.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Decompress.cs @@ -4,6 +4,7 @@ using System.IO; using System.IO.Compression; using System.Reflection; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -81,9 +82,10 @@ public static bool IsCompressed (string path) /// The zip to search in /// The relative path inside the zip to extract (may be a file or a directory). /// The location on disk to store the extracted results + /// The cancellation token (if any= /// The location on disk to the extracted resource /// - public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List createdFiles, [NotNullWhen (true)] out string? decompressedResource) + public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List createdFiles, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? decompressedResource) { decompressedResource = Path.Combine (decompressionDir, resource); @@ -101,11 +103,11 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso bool rv; if (Environment.OSVersion.Platform == PlatformID.Win32NT) { - rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir); + rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken); } else if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("XAMARIN_USE_SYSTEM_IO_COMPRESSION"))) { - rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir); + rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken); } else { - rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir); + rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir, cancellationToken); } if (rv) { @@ -128,7 +130,7 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso // The dir separator character in zip files is always "/", even on Windows const char zipDirectorySeparator = '/'; - static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir) + static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken) { Directory.CreateDirectory (decompressionDir); var args = new List { @@ -157,11 +159,11 @@ static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string r args.Add (zipPattern); } - var rv = XamarinTask.ExecuteAsync (log, "unzip", args).Result; + var rv = XamarinTask.ExecuteAsync (log, "unzip", args, cancellationToken: cancellationToken).Result; return rv.ExitCode == 0; } - static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir) + static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken) { var rv = true; @@ -172,6 +174,7 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string using var archive = ZipFile.OpenRead (zip); foreach (var entry in archive.Entries) { + cancellationToken?.ThrowIfCancellationRequested (); var entryPath = entry.FullName; if (entryPath.Length == 0) continue; @@ -207,7 +210,11 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string Directory.CreateDirectory (Path.GetDirectoryName (targetPath)); using var streamWrite = File.OpenWrite (targetPath); using var streamRead = entry.Open (); - streamRead.CopyTo (streamWrite); +#if NET + streamRead.CopyToAsync (streamWrite, cancellationToken ?? CancellationToken.None).Wait (); +#else + streamRead.CopyToAsync (streamWrite, 81920 /* default buffer size according to docs */, cancellationToken ?? CancellationToken.None).Wait (); +#endif log.LogMessage (MessageImportance.Low, "Extracted {0} into {1}", entryPath, targetPath); } } diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferences.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferences.cs index 0d39e8e91a00..a0fbdca8bc69 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferences.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferences.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Threading; using System.Xml; using Microsoft.Build.Framework; @@ -70,6 +71,8 @@ public class ResolveNativeReferences : XamarinTask, ITaskCallback { #endregion + CancellationTokenSource? cancellationTokenSource; + string GetIntermediateDecompressionDir (ITaskItem item) { return GetIntermediateDecompressionDir (item.ItemSpec); @@ -112,9 +115,11 @@ bool ExecuteLocally () var native_frameworks = new List (); var createdFiles = new List (); + cancellationTokenSource = new CancellationTokenSource (); + // there can be direct native references inside a project foreach (var nr in NativeReferences) { - ProcessNativeReference (nr, native_frameworks, createdFiles); + ProcessNativeReference (nr, native_frameworks, createdFiles, cancellationTokenSource.Token); } // or (managed) reference to an assembly that bind a framework @@ -122,11 +127,11 @@ bool ExecuteLocally () // look for sidecar's manifest var resources = Path.ChangeExtension (r.ItemSpec, ".resources"); if (Directory.Exists (resources)) { - ProcessNativeReference (r, resources, native_frameworks, createdFiles); + ProcessNativeReference (r, resources, native_frameworks, createdFiles, cancellationTokenSource.Token); } else { resources = resources + ".zip"; if (File.Exists (resources)) - ProcessNativeReference (r, resources, native_frameworks, createdFiles); + ProcessNativeReference (r, resources, native_frameworks, createdFiles, cancellationTokenSource.Token); } } @@ -136,12 +141,12 @@ bool ExecuteLocally () return !Log.HasLoggedErrors; } - void ProcessNativeReference (ITaskItem item, List native_frameworks, List createdFiles) + void ProcessNativeReference (ITaskItem item, List native_frameworks, List createdFiles, CancellationToken? cancellationTokenSource) { - ProcessNativeReference (item, item.ItemSpec, native_frameworks, createdFiles); + ProcessNativeReference (item, item.ItemSpec, native_frameworks, createdFiles, cancellationTokenSource); } - void ProcessNativeReference (ITaskItem item, string name, List native_frameworks, List createdFiles) + void ProcessNativeReference (ITaskItem item, string name, List native_frameworks, List createdFiles, CancellationToken? cancellationToken) { // '.' can be used to represent a file (instead of the directory) if (Path.GetFileName (name) == ".") @@ -190,7 +195,7 @@ void ProcessNativeReference (ITaskItem item, string name, List native // (compressed) xcframework if (name.EndsWith (".xcframework", StringComparison.OrdinalIgnoreCase) || name.EndsWith (".xcframework.zip", StringComparison.OrdinalIgnoreCase)) { - if (!TryResolveXCFramework (Log, TargetFrameworkMoniker, SdkIsSimulator, Architectures, name, GetIntermediateDecompressionDir (item), createdFiles, out var nativeLibraryPath)) + if (!TryResolveXCFramework (Log, TargetFrameworkMoniker, SdkIsSimulator, Architectures, name, GetIntermediateDecompressionDir (item), createdFiles, cancellationToken, out var nativeLibraryPath)) return; var nr = new TaskItem (item); SetMetadataNativeLibrary (nr, nativeLibraryPath); @@ -200,7 +205,7 @@ void ProcessNativeReference (ITaskItem item, string name, List native // compressed framework if (name.EndsWith (".framework.zip", StringComparison.OrdinalIgnoreCase)) { - if (!CompressionHelper.TryDecompress (Log, name, Path.GetFileNameWithoutExtension (name), GetIntermediateDecompressionDir (item), createdFiles, out var frameworkPath)) + if (!CompressionHelper.TryDecompress (Log, name, Path.GetFileNameWithoutExtension (name), GetIntermediateDecompressionDir (item), createdFiles, cancellationToken, out var frameworkPath)) return; var nr = new TaskItem (item); nr.ItemSpec = GetActualLibrary (frameworkPath); @@ -213,13 +218,13 @@ void ProcessNativeReference (ITaskItem item, string name, List native // sidecar / binding resource package if (name.EndsWith (".resources", StringComparison.OrdinalIgnoreCase)) { - ProcessSidecar (item, name, native_frameworks, createdFiles); + ProcessSidecar (item, name, native_frameworks, createdFiles, cancellationToken); return; } // compressed sidecar / binding resource package if (name.EndsWith (".resources.zip", StringComparison.OrdinalIgnoreCase)) { - ProcessSidecar (item, name, native_frameworks, createdFiles); + ProcessSidecar (item, name, native_frameworks, createdFiles, cancellationToken); return; } @@ -289,7 +294,7 @@ void SetMetadataNativeLibrary (ITaskItem item, string nativeLibraryPath) item.SetMetadata ("RelativePath", Path.Combine (FrameworksDirectory, Path.GetFileName (Path.GetDirectoryName (item.ItemSpec)))); } - void ProcessSidecar (ITaskItem r, string resources, List native_frameworks, List createdFiles) + void ProcessSidecar (ITaskItem r, string resources, List native_frameworks, List createdFiles, CancellationToken? cancellationToken) { if (!TryGetSidecarManifest (Log, resources, out var manifestContents)) return; @@ -302,7 +307,7 @@ void ProcessSidecar (ITaskItem r, string resources, List native_frame var name = referenceNode.Attributes ["Name"].Value.Trim ('\\', '/'); switch (Path.GetExtension (name)) { case ".xcframework": { - if (!TryResolveXCFramework (Log, TargetFrameworkMoniker, SdkIsSimulator, Architectures, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, out var nativeLibraryPath)) + if (!TryResolveXCFramework (Log, TargetFrameworkMoniker, SdkIsSimulator, Architectures, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, cancellationToken, out var nativeLibraryPath)) continue; SetMetadataNativeLibrary (t, nativeLibraryPath); break; @@ -311,7 +316,7 @@ void ProcessSidecar (ITaskItem r, string resources, List native_frame string? frameworkPath; if (!isCompressed) { frameworkPath = Path.Combine (resources, name); - } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, out frameworkPath)) { + } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, cancellationToken, out frameworkPath)) { continue; } t.ItemSpec = GetActualLibrary (frameworkPath); @@ -324,7 +329,7 @@ void ProcessSidecar (ITaskItem r, string resources, List native_frame string? dylibPath; if (!isCompressed) { dylibPath = Path.Combine (resources, name); - } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, out dylibPath)) { + } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, cancellationToken, out dylibPath)) { continue; } t.ItemSpec = dylibPath; @@ -335,7 +340,7 @@ void ProcessSidecar (ITaskItem r, string resources, List native_frame string? aPath; if (!isCompressed) { aPath = Path.Combine (resources, name); - } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, out aPath)) { + } else if (!CompressionHelper.TryDecompress (Log, resources, name, GetIntermediateDecompressionDir (resources), createdFiles, cancellationToken, out aPath)) { continue; } t.ItemSpec = aPath; @@ -374,7 +379,7 @@ void ProcessSidecar (ITaskItem r, string resources, List native_frame /// A full path to the resolved native library within the xcframework. If 'resourcePath' is compressed, this will point to where the native library is decompressed on disk. /// /// True if a native library was successfully found. Otherwise false, and an error will have been printed to the log. - public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFrameworkMoniker, bool isSimulator, string? architectures, string path, string intermediateDecompressionDir, List createdFiles, [NotNullWhen (true)] out string? nativeLibraryPath) + public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFrameworkMoniker, bool isSimulator, string? architectures, string path, string intermediateDecompressionDir, List createdFiles, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? nativeLibraryPath) { string resourcePath; string xcframework; @@ -386,7 +391,7 @@ public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFr resourcePath = Path.GetDirectoryName (path); xcframework = Path.GetFileName (path); } - return TryResolveXCFramework (log, targetFrameworkMoniker, isSimulator, architectures, resourcePath, xcframework, intermediateDecompressionDir, createdFiles, out nativeLibraryPath); + return TryResolveXCFramework (log, targetFrameworkMoniker, isSimulator, architectures, resourcePath, xcframework, intermediateDecompressionDir, createdFiles, cancellationToken, out nativeLibraryPath); } /// @@ -401,7 +406,7 @@ public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFr /// A full path to the resolved native library within the xcframework. If 'resourcePath' is compressed, this will point to where the native library is decompressed on disk. /// /// True if a native library was successfully found. Otherwise false, and an error will have been printed to the log. - public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFrameworkMoniker, bool isSimulator, string? architectures, string resourcePath, string xcframework, string intermediateDecompressionDir, List createdFiles, [NotNullWhen (true)] out string? nativeLibraryPath) + public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFrameworkMoniker, bool isSimulator, string? architectures, string resourcePath, string xcframework, string intermediateDecompressionDir, List createdFiles, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? nativeLibraryPath) { nativeLibraryPath = null; @@ -411,7 +416,7 @@ public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFr var isCompressed = CompressionHelper.IsCompressed (resourcePath); var xcframeworkPath = isCompressed ? resourcePath : Path.Combine (resourcePath, xcframework); - if (!TryResolveXCFramework (log, plist, xcframeworkPath, targetFrameworkMoniker, isSimulator, architectures!, out var nativeLibraryRelativePath)) + if (!TryResolveXCFramework (log, plist, xcframeworkPath, targetFrameworkMoniker, isSimulator, architectures!, cancellationToken, out var nativeLibraryRelativePath)) return false; if (!isCompressed) { @@ -420,7 +425,7 @@ public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFr } var zipResource = Path.Combine (xcframework, Path.GetDirectoryName (nativeLibraryRelativePath)); - if (!CompressionHelper.TryDecompress (log, resourcePath, zipResource, intermediateDecompressionDir, createdFiles, out var decompressedPath)) + if (!CompressionHelper.TryDecompress (log, resourcePath, zipResource, intermediateDecompressionDir, createdFiles, cancellationToken, out var decompressedPath)) return false; nativeLibraryPath = Path.Combine (intermediateDecompressionDir, xcframework, nativeLibraryRelativePath); @@ -444,7 +449,7 @@ public static bool TryResolveXCFramework (TaskLoggingHelper log, string targetFr /// The target architectures /// A relative path to the resolved native library within the xcframework. /// True if a native library was successfully found. Otherwise false, and an error will have been printed to the log. - internal static bool TryResolveXCFramework (TaskLoggingHelper log, PDictionary plist, string xcframeworkPath, string targetFrameworkMoniker, bool isSimulator, string architectures, [NotNullWhen (true)] out string? nativeLibraryPath) + internal static bool TryResolveXCFramework (TaskLoggingHelper log, PDictionary plist, string xcframeworkPath, string targetFrameworkMoniker, bool isSimulator, string architectures, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? nativeLibraryPath) { nativeLibraryPath = null; var platform = PlatformFrameworkHelper.GetFramework (targetFrameworkMoniker); @@ -519,8 +524,11 @@ internal static bool TryResolveXCFramework (TaskLoggingHelper log, PDictionary p public void Cancel () { - if (ShouldExecuteRemotely ()) + if (ShouldExecuteRemotely ()) { BuildConnection.CancelAsync (BuildEngine4).Wait (); + } else { + cancellationTokenSource?.Cancel (); + } } public bool ShouldCopyToBuildServer (ITaskItem item) => true; diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs index 6dc4ced90781..612af13b9989 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -38,25 +39,34 @@ public class Unzip : XamarinTask, ITaskCallback { [Output] public ITaskItem [] TouchedFiles { get; set; } = Array.Empty (); + CancellationTokenSource? cancellationTokenSource; public override bool Execute () { if (ShouldExecuteRemotely ()) { + Log.LogWarning ($"Unzip.Execute () about to execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); var rv = taskRunner.RunAsync (this).Result; if (rv && CopyToWindows) CopyFilesToWindowsAsync (taskRunner, TouchedFiles).Wait (); + Log.LogWarning ($"Unzip.Execute () executed remotely: {rv}"); return rv; } - return ExecuteLocally (); + Log.LogWarning ($"Unzip.Execute () about to execute locally"); + var rv2 = ExecuteLocally (); + Log.LogWarning ($"Unzip.Execute () executed locally"); + return rv2; } public void Cancel () { - if (ShouldExecuteRemotely ()) + if (ShouldExecuteRemotely ()) { BuildConnection.CancelAsync (BuildEngine4).Wait (); + } else { + cancellationTokenSource?.Cancel (); + } } public bool ShouldCopyToBuildServer (ITaskItem item) => true; @@ -73,7 +83,8 @@ public bool ShouldCreateOutputFile (ITaskItem item) bool ExecuteLocally () { var createdFiles = new List (); - if (!CompressionHelper.TryDecompress (Log, ZipFilePath!.ItemSpec, Resource, ExtractionPath, createdFiles, out var _)) + cancellationTokenSource = new CancellationTokenSource (); + if (!CompressionHelper.TryDecompress (Log, ZipFilePath!.ItemSpec, Resource, ExtractionPath, createdFiles, cancellationTokenSource.Token, out var _)) return false; TouchedFiles = createdFiles.Select (v => new TaskItem (v)).ToArray (); diff --git a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ResolveNativeReferencesTaskTest.cs b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ResolveNativeReferencesTaskTest.cs index 838d1be627ce..e4cf9232bdf3 100644 --- a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ResolveNativeReferencesTaskTest.cs +++ b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ResolveNativeReferencesTaskTest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Threading; using Microsoft.Build.Utilities; using NUnit.Framework; @@ -40,7 +41,7 @@ public void Xcode12_x (string targetFrameworkMoniker, bool isSimulator, string a // on Xcode 12.2+ you get arm64 for all (iOS, tvOS and watchOS) simulators var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "xcf-xcode12.2.plist"); var plist = PDictionary.FromFile (path)!; - var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, out var frameworkPath); + var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, null, out var frameworkPath); Assert.AreEqual (result, !string.IsNullOrEmpty (expected), "result"); Assert.That (frameworkPath, Is.EqualTo (expected), "frameworkPath"); } @@ -53,7 +54,7 @@ public void PreXcode12 (string targetFrameworkMoniker, bool isSimulator, string { var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "xcf-prexcode12.plist"); var plist = PDictionary.FromFile (path)!; - var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, out var frameworkPath); + var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, null, out var frameworkPath); Assert.AreEqual (result, !string.IsNullOrEmpty (expected), "result"); Assert.That (frameworkPath, Is.EqualTo (expected), "frameworkPath"); } @@ -62,7 +63,7 @@ public void PreXcode12 (string targetFrameworkMoniker, bool isSimulator, string public void BadInfoPlist () { var plist = new PDictionary (); - var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", TargetFramework.DotNet_iOS_String, false, "x86_64", out var frameworkPath); + var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", TargetFramework.DotNet_iOS_String, false, "x86_64", null, out var frameworkPath); Assert.IsFalse (result, "Invalid Info.plist"); } } From fe2308e1f6ac77e94ff8f2c6c9a575cd7d4bf3b4 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 26 Sep 2024 13:50:49 +0200 Subject: [PATCH 17/59] Debug spew --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index b70c7bbafd5e..5f43ed3d8db8 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -52,8 +52,15 @@ public override bool Execute () taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! }); - Log.LogWarning ($"Ditto.Execute () executed remotely"); - return taskRunner.RunAsync (this).Result; + var rv = taskRunner.RunAsync (this).Result; + Log.LogWarning ($"Ditto.Execute () executed remotely: {rv}"); + return rv; + } + + var src = Source!.ItemSpec; + if (!File.Exists (src) && !Directory.Exists (src)) { + Log.LogError ($"The source {src} does not exist."); + return false; } var args = new List (); From 7e8aa48d8e9879de747b9dcc5a331901d22b9351 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 27 Sep 2024 16:22:02 +0200 Subject: [PATCH 18/59] [msbuild] Fix Ditto task. --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index 5f43ed3d8db8..5d8d55636ec6 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -13,7 +13,7 @@ using Xamarin.Utils; namespace Xamarin.MacDev.Tasks { - public class Ditto : XamarinTask, ICancelableTask { + public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { #region Inputs public string? AdditionalArguments { get; set; } From f81f8314ad55e11f1e36d1679beb9508cb5bbeb6 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 1 Oct 2024 01:04:08 +0200 Subject: [PATCH 19/59] More logging --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index 5d8d55636ec6..ad3ff8742fc1 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -47,11 +47,12 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { public override bool Execute () { if (ShouldExecuteRemotely ()) { - Log.LogWarning ($"Ditto.Execute () about to execute remotely"); + Log.LogWarning ($"Ditto.Execute () will execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! }); + Log.LogWarning ($"Ditto.Execute () about to execute remotely"); var rv = taskRunner.RunAsync (this).Result; Log.LogWarning ($"Ditto.Execute () executed remotely: {rv}"); return rv; From a93bbc1f42a725f3cdc0e1b9ab5b1d1aecef9a22 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 1 Oct 2024 01:04:30 +0200 Subject: [PATCH 20/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 19a91ff4c472..d8bb307aac33 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.36-diagnose-cancelled-posts] + [2.1.37-diagnose-cancelled-posts] [1.1.7] From 62b3985ee02def3d80a00b61fff1118d5ab30040 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 1 Oct 2024 00:23:02 +0200 Subject: [PATCH 21/59] =?UTF-8?q?[build]=C2=A0Shutdown/kill=20any=20build?= =?UTF-8?q?=20servers=20at=20the=20end=20of=20the=20build.=20(#21315)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a log from our bots, note the 14 minute gap just before printing the timing results: ``` [...] 2024-09-27T07:34:00.3958920Z Making install in dotnet 2024-09-27T07:34:01.7633820Z Validated file permissions for Xamarin.Mac. 2024-09-27T07:34:01.7800150Z Validated file permissions for Xamarin.iOS. 2024-09-27T07:34:01.7825300Z 2024-09-27T07:34:01.7872490Z Xamarin.iOS has not been installed into your system by 'make install' 2024-09-27T07:34:01.7918570Z In order to set the currently built Xamarin.iOS as your system version, 2024-09-27T07:34:01.7965090Z execute 'make install-system'. 2024-09-27T07:34:01.7987920Z 2024-09-27T07:34:01.8034290Z Xamarin.Mac has not been installed into your system by 'make install' 2024-09-27T07:34:01.8080260Z In order to set the currently built Xamarin.Mac as your system version, 2024-09-27T07:34:01.8126200Z execute 'make install-system'. 2024-09-27T07:34:01.8148530Z 2024-09-27T07:48:22.3100850Z 2024-09-27T07:48:22.3102130Z real 15m26.160s 2024-09-27T07:48:22.3102800Z user 1m4.044s 2024-09-27T07:48:22.3103270Z sys 0m18.379s ``` What happens is this: * We're using parallel make, and parallel make will start a jobserver, managed by file descriptors, where these file descriptors must be closed in all subprocesses for make to realize it's done. * Any 'dotnet build' might start a build server * The build server does not close any file descriptors it may have inherited when daemonizing itself. * Thus the build server (which will still be alive after we're done building here) might have a file descriptor open which make is waiting for. * The proper fix is to fix the build server to close its file descriptors. * The intermediate working is to shut down the build server instead. This will save 10-15 minutes at the end of every build in the bots. --- Makefile | 1 + dotnet/Makefile | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef263bb0ceaa..5f79a6024671 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,7 @@ else @echo endif endif + $(Q) $(MAKE) -C dotnet shutdown-build-server .PHONY: package release package release: diff --git a/dotnet/Makefile b/dotnet/Makefile index 6e54ef326dcf..7e1eb766c266 100644 --- a/dotnet/Makefile +++ b/dotnet/Makefile @@ -550,4 +550,9 @@ all-hook:: $(Q) $(MAKE) shutdown-build-server shutdown-build-server: - $(Q) $(DOTNET) build-server shutdown + $(Q) echo "Shutting down build servers:" + $(Q) $(DOTNET) build-server shutdown | sed 's/^/ /' || true + $(Q) echo "Listing .NET processes still alive:" + $(Q) pgrep -lf "^$(DOTNET)" | sed 's/^/ /' || true + $(Q) echo "Killing the above mentioned processes." + $(Q) pkill -9 -f "^$(DOTNET)" | sed 's/^/ /' || true From 198ca618111103b8bfd8c0e01db30fc788b4f469 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 1 Oct 2024 12:19:42 +0200 Subject: [PATCH 22/59] Stack trace --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index ad3ff8742fc1..49a5cf94ee97 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -80,7 +80,7 @@ public override bool Execute () } } - Log.LogWarning ($"Ditto.Execute () about to execute locally"); + Log.LogWarning ($"Ditto.Execute () about to execute locally\n{Environment.StackTrace}"); cancellationTokenSource = new CancellationTokenSource (); cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait (); From ea85e732d65c3330284a96510ef53c17a0ddc780 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 1 Oct 2024 17:52:41 +0200 Subject: [PATCH 23/59] More debug spew. --- msbuild/Directory.Build.props | 2 +- .../Handlers/ExecuteTaskMessageHandler.cs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index d8bb307aac33..628ad97d5e9d 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.37-diagnose-cancelled-posts] + [2.1.39-diagnose-cancelled-posts] [1.1.7] diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs index 6e905dcbb12c..6c02d35a03b9 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs +++ b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs @@ -25,6 +25,10 @@ public ExecuteTaskMessageHandler () protected override async Task ExecuteAsync (ExecuteTaskMessage message) { + var msg = $"{global::System.DateTime.UtcNow.ToString ("o")} ExecuteTaskMessageHandler.ExecuteAsync ({message.TaskName})\n{global::System.Environment.StackTrace}"; + Console.WriteLine (msg); + Console.Error.WriteLine (msg); + tracer.Info (msg); return await Task.Run (() => { // We need to lock in order to change the current directory lock (lockObject) { From 760537ed9485bb85adb7f0af9845f95e0ad5180b Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 2 Oct 2024 10:54:20 +0200 Subject: [PATCH 24/59] Speed things up a bit. --- Make.config | 8 ++-- .../pipelines/run-macos-tests-pipeline.yml | 44 +------------------ .../automation/templates/tests-stage.yml | 36 --------------- 3 files changed, 5 insertions(+), 83 deletions(-) diff --git a/Make.config b/Make.config index 1cf09339a6a8..141adf75f71f 100644 --- a/Make.config +++ b/Make.config @@ -317,10 +317,10 @@ MIN_TVOS_SIMULATOR_VERSION=15.0 EXTRA_SIMULATORS=com.apple.pkg.iPhoneSimulatorSDK15_0 com.apple.pkg.AppleTVSimulatorSDK15_0 com.apple.pkg.WatchSimulatorSDK8_0 INCLUDE_IOS=1 -INCLUDE_MAC=1 -INCLUDE_WATCH=1 -INCLUDE_TVOS=1 -INCLUDE_MACCATALYST=1 +INCLUDE_MAC= +INCLUDE_WATCH= +INCLUDE_TVOS= +INCLUDE_MACCATALYST= INCLUDE_DEVICE=1 INCLUDE_DOTNET_WATCHOS= INCLUDE_XAMARIN_LEGACY= diff --git a/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml b/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml index 1bf2a4b2c6d3..7ef2ecd983b8 100644 --- a/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml +++ b/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml @@ -21,49 +21,7 @@ parameters: - name: macTestsConfigurations displayName: macOS test configurations to run type: object - default: [ - { - stageName: 'mac_12_m1', - displayName: 'M1 - Mac Ventura (12)', - macPool: 'VSEng-VSMac-Xamarin-Shared', - useImage: false, - statusContext: 'M1 - Mac Monterey (12)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Monterey", - "macOS.Architecture -equals arm64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }, - { - stageName: 'mac_13_m1', - displayName: 'M1 - Mac Ventura (13)', - macPool: 'VSEng-VSMac-Xamarin-Shared', - useImage: false, - statusContext: 'M1 - Mac Ventura (13)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Ventura", - "macOS.Architecture -equals arm64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }, - { - stageName: 'mac_14_x64', - displayName: 'X64 - Mac Sonoma (14)', - macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted', - useImage: false, - statusContext: 'X64 - Mac Sonoma (14)', - demands: [ - "Agent.OS -equals Darwin", - "macOS.Name -equals Sonoma", - "macOS.Architecture -equals x64", - "Agent.HasDevices -equals False", - "Agent.IsPaired -equals False" - ] - }] + default: [] - name: pool type: string diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index e54a23ee9cd7..cd4fec474c41 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -218,42 +218,6 @@ stages: statusContext: 'VSTS: simulator tests' uploadArtifacts: true -# always run simulator tests -- template: ./tests/stage.yml - parameters: - xcodeChannel: ${{ parameters.xcodeChannel }} - macOSName: ${{ parameters.macOSName }} - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} - testConfigurations: ${{ parameters.testConfigurations }} - supportedPlatforms: ${{ parameters.supportedPlatforms }} - stageName: 'simulator_tests' - displayName: '${{ parameters.stageDisplayNamePrefix }}Simulator tests' - testPool: '' # use the default - statusContext: 'VSTS: simulator tests' - makeTarget: 'jenkins' - vsdropsPrefix: ${{ variables.vsdropsPrefix }} - keyringPass: $(pass--lab--mac--builder--keychain) - gitHubToken: $(Github.Token) - xqaCertPass: $(xqa--certificates--password) - condition: ${{ parameters.runTests }} - postPipeline: true - -- template: ./tests/publish-results.yml - parameters: - displayName: '${{ parameters.stageDisplayNamePrefix }}Publish Test Results' - stageName: 'publish_test_results' - statusContext: 'VSTS: test results' - vsdropsPrefix: ${{ variables.vsdropsPrefix }} - condition: ${{ parameters.runTests }} - testConfigurations: ${{ parameters.testConfigurations }} - supportedPlatforms: ${{ parameters.supportedPlatforms }} - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} - postPipeline: true - - ${{ if eq(parameters.runWindowsIntegration, true) }}: - template: ./windows/stage.yml parameters: From f8a39d79c94f28c8663c91d58dc556bcab61a23b Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 2 Oct 2024 10:59:21 +0200 Subject: [PATCH 25/59] Fix build. --- .../Handlers/ExecuteTaskMessageHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs index 6c02d35a03b9..e44efbe1eacf 100644 --- a/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs +++ b/msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs @@ -1,3 +1,4 @@ +using System; using System.Diagnostics; using System.IO; using System.Threading.Tasks; @@ -25,7 +26,7 @@ public ExecuteTaskMessageHandler () protected override async Task ExecuteAsync (ExecuteTaskMessage message) { - var msg = $"{global::System.DateTime.UtcNow.ToString ("o")} ExecuteTaskMessageHandler.ExecuteAsync ({message.TaskName})\n{global::System.Environment.StackTrace}"; + var msg = $"{System.DateTime.UtcNow.ToString ("o")} ExecuteTaskMessageHandler.ExecuteAsync ({message.TaskName})\n{System.Environment.StackTrace}"; Console.WriteLine (msg); Console.Error.WriteLine (msg); tracer.Info (msg); From ebcbfa68cf51002043eb71466edc6842a5767f6a Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 2 Oct 2024 15:34:31 +0200 Subject: [PATCH 26/59] More debug spew. --- msbuild/Directory.Build.props | 2 +- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 628ad97d5e9d..30d4de4791a0 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.39-diagnose-cancelled-posts] + [2.1.41-diagnose-cancelled-posts] [1.1.7] diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index 49a5cf94ee97..ff1fb6ed7495 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -44,17 +44,24 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { CancellationTokenSource? cancellationTokenSource; + void LogLine (string msg) + { + Log.LogWarning (msg); + Console.WriteLine ($"{DateTime.UtcNow.ToString("o")} Ditto stdout: {msg}"); + Console.Error.WriteLine ($"{DateTime.UtcNow.ToString("o")} Ditto stderr: {msg}"); + } + public override bool Execute () { if (ShouldExecuteRemotely ()) { - Log.LogWarning ($"Ditto.Execute () will execute remotely"); + LogLine ($"Ditto.Execute () will execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! }); - Log.LogWarning ($"Ditto.Execute () about to execute remotely"); + LogLine ($"Ditto.Execute () about to execute remotely"); var rv = taskRunner.RunAsync (this).Result; - Log.LogWarning ($"Ditto.Execute () executed remotely: {rv}"); + LogLine ($"Ditto.Execute () executed remotely: {rv}"); return rv; } @@ -80,11 +87,11 @@ public override bool Execute () } } - Log.LogWarning ($"Ditto.Execute () about to execute locally\n{Environment.StackTrace}"); + LogLine ($"Ditto.Execute () about to execute locally\n{Environment.StackTrace}"); cancellationTokenSource = new CancellationTokenSource (); cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait (); - Log.LogWarning ($"Ditto.Execute () executed locally"); + LogLine ($"Ditto.Execute () executed locally"); // Create a list of all the files we've copied var copiedFiles = new List (); From a3c011042f69ed93969d33021d4a70b3a35ae757 Mon Sep 17 00:00:00 2001 From: GitHub Actions Autoformatter Date: Thu, 3 Oct 2024 10:37:08 +0000 Subject: [PATCH 27/59] Auto-format source code --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index ff1fb6ed7495..7b8ea3330cee 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -47,8 +47,8 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { void LogLine (string msg) { Log.LogWarning (msg); - Console.WriteLine ($"{DateTime.UtcNow.ToString("o")} Ditto stdout: {msg}"); - Console.Error.WriteLine ($"{DateTime.UtcNow.ToString("o")} Ditto stderr: {msg}"); + Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); + Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); } public override bool Execute () From a2d19687cac1ef5b291ebeac07f6fcd82f065c80 Mon Sep 17 00:00:00 2001 From: Rolf Kvinge Date: Thu, 3 Oct 2024 12:55:31 +0200 Subject: [PATCH 28/59] devops: make .net path overridable --- tools/devops/automation/scripts/run-remote-windows-tests.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/devops/automation/scripts/run-remote-windows-tests.ps1 b/tools/devops/automation/scripts/run-remote-windows-tests.ps1 index e90cc2457028..41d09cd8d77a 100644 --- a/tools/devops/automation/scripts/run-remote-windows-tests.ps1 +++ b/tools/devops/automation/scripts/run-remote-windows-tests.ps1 @@ -1,4 +1,6 @@ -$Env:DOTNET = "$Env:BUILD_SOURCESDIRECTORY\xamarin-macios\tests\dotnet\Windows\bin\dotnet\dotnet.exe" +if ("$Env:DOTNET" -eq "") { + $Env:DOTNET = "$Env:BUILD_SOURCESDIRECTORY\xamarin-macios\tests\dotnet\Windows\bin\dotnet\dotnet.exe" +} $Env:ServerAddress = $Env:MAC_AGENT_IP $Env:ServerUser = $Env:MAC_AGENT_USER $Env:ServerPassword = $Env:XMA_PASSWORD From 83127dfb0515b0277f11e770c0dcfc8721cd902e Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 3 Oct 2024 18:08:02 +0200 Subject: [PATCH 29/59] More debug spew. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 30d4de4791a0..1113cbf12370 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.41-diagnose-cancelled-posts] + [2.1.43-diagnose-cancelled-posts] [1.1.7] From fa6ff3433e9f522d1bd0961c0480e3101b3078ab Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 3 Oct 2024 20:59:14 +0200 Subject: [PATCH 30/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 1113cbf12370..f1baa2462a5b 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.43-diagnose-cancelled-posts] + [2.1.45-diagnose-cancelled-posts] [1.1.7] From 13d3609cf2f06fdfbaa58df7b939623195844e99 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 4 Oct 2024 11:05:20 +0200 Subject: [PATCH 31/59] [msbuild] Make MessagingVersion overridable from the environment. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index f1baa2462a5b..2521d8377c13 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.45-diagnose-cancelled-posts] + [2.1.45-diagnose-cancelled-posts] [1.1.7] From ec3c83eb3c1a0e906fc834c4f8a746f273292bb2 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 4 Oct 2024 16:57:41 +0200 Subject: [PATCH 32/59] [devops] Make the prepare-for-remote-tests.sh script callable locally. --- tools/devops/automation/scripts/prepare-for-remote-tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/scripts/prepare-for-remote-tests.sh b/tools/devops/automation/scripts/prepare-for-remote-tests.sh index bf60da1366dc..f7a3aa1495aa 100755 --- a/tools/devops/automation/scripts/prepare-for-remote-tests.sh +++ b/tools/devops/automation/scripts/prepare-for-remote-tests.sh @@ -1,10 +1,12 @@ #!/bin/bash -eux +cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." + # Install the local .NET we're using into XMA's directory # (we can't point XMA to our local directory) mkdir -p ~/Library/Caches/Xamarin/XMA/SDKs -cp -cRH "$BUILD_SOURCESDIRECTORY"/xamarin-macios/builds/downloads/dotnet ~/Library/Caches/Xamarin/XMA/SDKs -sed '/local-tests-feed/d' "$BUILD_SOURCESDIRECTORY"/xamarin-macios/NuGet.config > ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/NuGet.config +cp -cRH ./builds/downloads/dotnet ~/Library/Caches/Xamarin/XMA/SDKs +sed '/local-tests-feed/d' ./NuGet.config > ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/NuGet.config mkdir -p ~/Library/Caches/Xamarin/XMA/SDKs/.home/.nuget/NuGet/ cp ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/NuGet.config ~/Library/Caches/Xamarin/XMA/SDKs/.home/.nuget/NuGet/NuGet.Config From 797900d5bc829beee47e5397dba747ed00ebb30f Mon Sep 17 00:00:00 2001 From: Rolf Kvinge Date: Fri, 4 Oct 2024 17:08:42 +0200 Subject: [PATCH 33/59] Don't try to build test libraries on Windows --- tests/bindings-xcframework-test/dotnet/shared.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bindings-xcframework-test/dotnet/shared.csproj b/tests/bindings-xcframework-test/dotnet/shared.csproj index c04de55618f9..8812763872d9 100644 --- a/tests/bindings-xcframework-test/dotnet/shared.csproj +++ b/tests/bindings-xcframework-test/dotnet/shared.csproj @@ -59,6 +59,6 @@ - + From 7685268a4ebdf99fdba98853deedd82ff5766d89 Mon Sep 17 00:00:00 2001 From: Rolf Kvinge Date: Fri, 4 Oct 2024 17:09:12 +0200 Subject: [PATCH 34/59] Ignore output that looks like errors. --- tests/dotnet/Windows/InstallDotNet.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dotnet/Windows/InstallDotNet.csproj b/tests/dotnet/Windows/InstallDotNet.csproj index 2f208f0f270d..79ceb5037866 100644 --- a/tests/dotnet/Windows/InstallDotNet.csproj +++ b/tests/dotnet/Windows/InstallDotNet.csproj @@ -171,6 +171,7 @@ Command=""$(DotNetToolPath)" workload install %(_WorkloadIds.Identity) --skip-manifest-update --verbosity diag --temp-dir "$(DotNetTempDirectory)" --configfile "$(MacIosRootDirectory)NuGet.config" --skip-sign-check" WorkingDirectory="$(MSBuildThisFileDirectory)" EnvironmentVariables="DOTNET_MULTILEVEL_LOOKUP=0" + IgnoreStandardErrorWarningFormat="true" /> From c140af3af3b7e7f90fa683f6733b3f8602220ed1 Mon Sep 17 00:00:00 2001 From: Rolf Kvinge Date: Fri, 4 Oct 2024 17:09:42 +0200 Subject: [PATCH 35/59] Add .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f152fefaf654..17c5e9fe3ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +config.runsettings Build.props Make.config.inc Make.config.local From f192a39213c5125b0f4b0abd39c224a1e4e74f5a Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 08:37:19 +0200 Subject: [PATCH 36/59] message, not warning --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 2 +- msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index 7b8ea3330cee..d08bf959a759 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -46,7 +46,7 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { void LogLine (string msg) { - Log.LogWarning (msg); + Log.LogMessage (MessageImportance.Low, (msg); Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); } diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs index 612af13b9989..0733f52f1b9b 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs @@ -39,24 +39,31 @@ public class Unzip : XamarinTask, ITaskCallback { [Output] public ITaskItem [] TouchedFiles { get; set; } = Array.Empty (); + void LogLine (string msg) + { + Log.LogMessage (MessageImportance.Low, (msg); + Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); + Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); + } + CancellationTokenSource? cancellationTokenSource; public override bool Execute () { if (ShouldExecuteRemotely ()) { - Log.LogWarning ($"Unzip.Execute () about to execute remotely"); + LogLine ($"Unzip.Execute () about to execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); var rv = taskRunner.RunAsync (this).Result; if (rv && CopyToWindows) CopyFilesToWindowsAsync (taskRunner, TouchedFiles).Wait (); - Log.LogWarning ($"Unzip.Execute () executed remotely: {rv}"); + LogLine ($"Unzip.Execute () executed remotely: {rv}"); return rv; } - Log.LogWarning ($"Unzip.Execute () about to execute locally"); + LogLine ($"Unzip.Execute () about to execute locally"); var rv2 = ExecuteLocally (); - Log.LogWarning ($"Unzip.Execute () executed locally"); + LogLine ($"Unzip.Execute () executed locally"); return rv2; } From 692e38692c4bfc9d4cab987b7679077e0b3c0543 Mon Sep 17 00:00:00 2001 From: Rolf Kvinge Date: Fri, 4 Oct 2024 17:22:19 +0200 Subject: [PATCH 37/59] Fix condition --- tests/bindings-xcframework-test/dotnet/shared.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bindings-xcframework-test/dotnet/shared.csproj b/tests/bindings-xcframework-test/dotnet/shared.csproj index 8812763872d9..af7801fc3a63 100644 --- a/tests/bindings-xcframework-test/dotnet/shared.csproj +++ b/tests/bindings-xcframework-test/dotnet/shared.csproj @@ -59,6 +59,6 @@ - + From f513526da8f950d4b398b5e066feb91748254e66 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 08:56:16 +0200 Subject: [PATCH 38/59] Fix build. --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 2 +- msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index d08bf959a759..5f75af19e2fe 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -46,7 +46,7 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { void LogLine (string msg) { - Log.LogMessage (MessageImportance.Low, (msg); + Log.LogMessage (MessageImportance.Low, msg); Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); } diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs index 0733f52f1b9b..272861ffce24 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs @@ -41,7 +41,7 @@ public class Unzip : XamarinTask, ITaskCallback { void LogLine (string msg) { - Log.LogMessage (MessageImportance.Low, (msg); + Log.LogMessage (MessageImportance.Low, msg); Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); } From efccf4e222a25f9164ad1c5d18273fb0b9f87fb9 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 09:49:22 +0200 Subject: [PATCH 39/59] Bump Xamarin.MacDev. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 2521d8377c13..11bcbf3d98ad 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.45-diagnose-cancelled-posts] + [2.1.49-diagnose-cancelled-posts] [1.1.7] From cc19f5cd4352f06bb105d8f4109d224f082fbda4 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 12:10:13 +0200 Subject: [PATCH 40/59] Fix version. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 11bcbf3d98ad..81ffad7cfa48 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.49-diagnose-cancelled-posts] + [2.1.49-pushpackage-diagnose-cancelled-posts] [1.1.7] From 9d4ab017a58ddf5624a8a1b65e3e9121f6a7e652 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 10:50:55 +0200 Subject: [PATCH 41/59] Bump and skip runtime check. --- msbuild/Directory.Build.props | 2 +- tools/devops/automation/scripts/run-remote-windows-tests.ps1 | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 81ffad7cfa48..81db942bb249 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.49-pushpackage-diagnose-cancelled-posts] + [2.1.50-pushpackage-diagnose-cancelled-posts] [1.1.7] diff --git a/tools/devops/automation/scripts/run-remote-windows-tests.ps1 b/tools/devops/automation/scripts/run-remote-windows-tests.ps1 index 41d09cd8d77a..b366a6536bfd 100644 --- a/tools/devops/automation/scripts/run-remote-windows-tests.ps1 +++ b/tools/devops/automation/scripts/run-remote-windows-tests.ps1 @@ -6,6 +6,8 @@ $Env:ServerUser = $Env:MAC_AGENT_USER $Env:ServerPassword = $Env:XMA_PASSWORD $Env:_DotNetRootRemoteDirectory = "/Users/$Env:MAC_AGENT_USER/Library/Caches/Xamarin/XMA/SDKs/dotnet/" +$Env:SkipCompatibleRuntimeCheck = "true" + & $Env:DOTNET ` test ` "$Env:BUILD_SOURCESDIRECTORY/xamarin-macios/tests/dotnet/UnitTests/DotNetUnitTests.csproj" ` From 850644e3c22a167796eecab8c7111f660d870288 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 16:34:56 +0200 Subject: [PATCH 42/59] Bump again. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 81db942bb249..606e82b7aea2 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.50-pushpackage-diagnose-cancelled-posts] + [2.1.51-pushpackage-diagnose-cancelled-posts] [1.1.7] From 82ae951abb0c3fb59a7ca82a747f2834990a2899 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 19:17:47 +0200 Subject: [PATCH 43/59] Bump again. --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 606e82b7aea2..efb4d0246192 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.51-pushpackage-diagnose-cancelled-posts] + [2.1.52-pushpackage-diagnose-cancelled-posts] [1.1.7] From 97d0f1cd5ff37ab1f70f7969b01806123fcd84fa Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 23:07:21 +0200 Subject: [PATCH 44/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index efb4d0246192..41a067c0a57a 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.52-pushpackage-diagnose-cancelled-posts] + [2.1.55-pushpackage-diagnose-cancelled-posts] [1.1.7] From 0b81eb9196861a683fd37d4036841dd66e2a2c1f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 23:37:07 +0200 Subject: [PATCH 45/59] Bump again. --- msbuild/Directory.Build.props | 2 +- tools/devops/automation/templates/windows/reserve-mac.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index 41a067c0a57a..de02b1be47d2 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.55-pushpackage-diagnose-cancelled-posts] + [2.1.57-pushpackage-diagnose-cancelled-posts] [1.1.7] diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 9c8710d9b619..14ea0d05447a 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -33,6 +33,9 @@ steps: - checkout: maccore persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail +- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 + displayName: 'Show Environment' + - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' name: disableCodeQLOnArm64 From 584354540731551eac87a6e421004106e8646514 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 7 Oct 2024 23:50:17 +0200 Subject: [PATCH 46/59] PR 21317 with a few extra bits. --- .../automation/scripts/bash/clean-bot.sh | 20 ++++++++++++- .../scripts/bash/clean-results-dir.sh | 4 --- .../scripts/bash/clean-simulator-runtime.sh | 5 ++++ .../scripts/bash/delete-library-dirs.sh | 6 ---- .../scripts/bash/kill-deadlocked-processes.sh | 27 +++++++++++++++++ .../run-generator-tests-on-windows.ps1 | 2 +- .../{show_env.ps1 => show_bot_info.ps1} | 29 ++++++++++++++----- .../automation/scripts/show_python_env.ps1 | 6 ---- .../build/api-diff-process-results.yml | 4 +-- .../automation/templates/common/configure.yml | 4 +-- .../templates/common/load_configuration.yml | 4 +-- .../automation/templates/common/setup.yml | 20 ++----------- .../devops/automation/templates/mac/build.yml | 21 +++++--------- .../automation/templates/tests/build.yml | 2 +- .../templates/tests/publish-html.yml | 4 +-- .../automation/templates/windows/build.yml | 7 ++--- .../templates/windows/reenable-mac.yml | 4 +-- .../templates/windows/reserve-mac.yml | 9 ++++-- 18 files changed, 103 insertions(+), 75 deletions(-) delete mode 100755 tools/devops/automation/scripts/bash/clean-results-dir.sh delete mode 100755 tools/devops/automation/scripts/bash/delete-library-dirs.sh create mode 100755 tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh rename tools/devops/automation/scripts/{show_env.ps1 => show_bot_info.ps1} (51%) delete mode 100644 tools/devops/automation/scripts/show_python_env.ps1 diff --git a/tools/devops/automation/scripts/bash/clean-bot.sh b/tools/devops/automation/scripts/bash/clean-bot.sh index 923d353c1a8d..c48e4d5895f7 100755 --- a/tools/devops/automation/scripts/bash/clean-bot.sh +++ b/tools/devops/automation/scripts/bash/clean-bot.sh @@ -12,6 +12,15 @@ df -h # We don't care about errors in this section, we just want to clean as much as possible set +e +# Clean workspace +( + REPO_PATH="SYSTEM_DEFAULTWORKINGDIRECTORY/$(basename "$BUILD_REPOSITORY_NAME")" + if test -d "$REPO_PATH"; then + cd "$REPO_PATH" + git clean -xfd + fi +) + # Delete all the simulator devices. These can take up a lot of space over time (I've seen 100+GB on the bots) /Applications/Xcode.app/Contents/Developer/usr/bin/simctl delete all @@ -118,7 +127,9 @@ XCODE_SELECT=$(xcode-select -p) for oldXcode in "${oldXcodes[@]}"; do if [ "$XCODE_SELECT" != "$oldXcode/Contents/Developer" ]; then - sudo rm -Rf "$oldXcode" + if test -d "$oldXcode"; then + sudo rm -Rf "$oldXcode" + fi else echo "Not removing $oldXcode because is the currently selected one." fi @@ -126,6 +137,13 @@ done DIR="$(dirname "${BASH_SOURCE[0]}")" "$DIR"/clean-simulator-runtime.sh +"$DIR"/kill-deadlocked-processes.sh + +# Remove legacy Xamarin/MonoTouch stuff +sudo rm -Rf /Developer/MonoTouch +sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework +sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework +ls -R /Library/Frameworks # Print disk status after cleaning df -h diff --git a/tools/devops/automation/scripts/bash/clean-results-dir.sh b/tools/devops/automation/scripts/bash/clean-results-dir.sh deleted file mode 100755 index 3e1aa2dfb4e1..000000000000 --- a/tools/devops/automation/scripts/bash/clean-results-dir.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -ex - -rm -Rvf package -time make -C xamarin-macios/ git-clean-all diff --git a/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh b/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh index ed23cf1193c2..dfddf64f38d9 100755 --- a/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh +++ b/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh @@ -5,6 +5,11 @@ set -o pipefail IFS=$'\n\t' +# delete all watchOS simulators, we don't need them anymore +for i in $(xcrun simctl runtime list | grep "watchOS.*Ready" | sed -e 's/.* - //' -e 's/ .*//'); do + xcrun simctl runtime delete "$i" +done + xcrun simctl runtime list -j > simruntime.json cat simruntime.json diff --git a/tools/devops/automation/scripts/bash/delete-library-dirs.sh b/tools/devops/automation/scripts/bash/delete-library-dirs.sh deleted file mode 100755 index 3c647824f72b..000000000000 --- a/tools/devops/automation/scripts/bash/delete-library-dirs.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -ex - -sudo rm -Rf /Developer/MonoTouch -sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework -sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework -ls -R /Library/Frameworks diff --git a/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh b/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh new file mode 100755 index 000000000000..7610d51a7200 --- /dev/null +++ b/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +echo "Looking for processes that have been stuck for more than a day, and will try to kill them." + +# Collect the list of processes for the current user, including the CPU time. +# We then split the CPU time into separate fields, so that it's easier to figure out the total number of minutes later on. +IFS=$'\n' +PROCESSES=() +while IFS='' read -r line; do PROCESSES+=("$line"); done < <(ps -o cputime=,pid=,user=,lstart=,args= -U "$USER" -w -w -w | sed -e 's/\([0-9]*\):\([0-9][0-9]\)\.\([0-9][0-9]\)/\1 m \2 s \3 ms/' | sort -nr) + +IFS=' ' +for process in "${PROCESSES[@]}"; do + IFS=" " read -r -a FIELDS <<< "$process" + minutes=${FIELDS[0]} + pid=${FIELDS[6]} + + echo "$process" + + # looking for processes that have spent more than a day using the CPU (24h * 60min = 1440min) + if (( minutes > 1440 )); then + echo " This process has been stuck for more than $minutes minutes, so assuming it's deadlocked and we'll try to kill it:" + echo " kill -9 $pid" + kill -9 "$pid" | sed 's/^/ /' || true + fi +done + +echo "No (more) processes stuck for more than a day." diff --git a/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 b/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 index 493b3026c0c2..a59e58043535 100644 --- a/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 +++ b/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 @@ -1,5 +1,5 @@ # Dump the environment to see what we're working with. -& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\show_env.ps1" +& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])\tools\devops\automation\scripts\show_bot_info.ps1" # Set a few variables $Env:DOTNET = "$Env:BUILD_SOURCESDIRECTORY\xamarin-macios\tests\dotnet\Windows\bin\dotnet\dotnet.exe" diff --git a/tools/devops/automation/scripts/show_env.ps1 b/tools/devops/automation/scripts/show_bot_info.ps1 similarity index 51% rename from tools/devops/automation/scripts/show_env.ps1 rename to tools/devops/automation/scripts/show_bot_info.ps1 index 7852d1ad7c7a..0369422be80e 100644 --- a/tools/devops/automation/scripts/show_env.ps1 +++ b/tools/devops/automation/scripts/show_bot_info.ps1 @@ -8,13 +8,14 @@ if ($IsMacOS -or $IsLinux) { Write-Host "COMPUTERNAME: ${env:COMPUTERNAME}" } -gci env: | format-table -autosize - -gci env: | format-table -autosize | Out-String -Width 8192 - -gci env: | format-table -autosize -wrap +Get-ChildItem env: | Sort-Object -Property Name | Format-Table -AutoSize | Out-String -Width 81920 if ($IsMacOS) { + Write-Host "" + Write-Host "## Uptime" + Write-Host "" + uptime + Write-Host "" Write-Host "## System profile" Write-Host "" @@ -25,10 +26,24 @@ if ($IsMacOS) { Write-Host "" ifconfig | grep 'inet ' + Write-Host "" + Write-Host "## Top processes (ps)" + Write-Host "" + ps aux + + Write-Host "" + Write-Host "## Python3 location:" + Write-Host "" + which python3 + + Write-Host "" + Write-Host "## Pip3 version:" + Write-Host "" + pip3 -V Write-Host "" - Write-Host "## Top processes" + Write-Host "## Hardware info" Write-Host "" - top -l 1 -o TIME + ioreg -l | grep -e Manufacturer -e 'Vendor Name' } diff --git a/tools/devops/automation/scripts/show_python_env.ps1 b/tools/devops/automation/scripts/show_python_env.ps1 deleted file mode 100644 index 79873375cf32..000000000000 --- a/tools/devops/automation/scripts/show_python_env.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -Write-Host "Python3 location" -which python3 - -Write-Host "Pip3 version" -pip3 -V - diff --git a/tools/devops/automation/templates/build/api-diff-process-results.yml b/tools/devops/automation/templates/build/api-diff-process-results.yml index c8b3bf3cc8bb..233054ab53cc 100644 --- a/tools/devops/automation/templates/build/api-diff-process-results.yml +++ b/tools/devops/automation/templates/build/api-diff-process-results.yml @@ -27,8 +27,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} - - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' + - pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) { diff --git a/tools/devops/automation/templates/common/configure.yml b/tools/devops/automation/templates/common/configure.yml index e909e535fe88..dadaa98f1dc1 100644 --- a/tools/devops/automation/templates/common/configure.yml +++ b/tools/devops/automation/templates/common/configure.yml @@ -65,8 +65,8 @@ steps: name: decisions displayName: 'Make decisions' -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 diff --git a/tools/devops/automation/templates/common/load_configuration.yml b/tools/devops/automation/templates/common/load_configuration.yml index 5ecac652b11f..f59efce0113f 100644 --- a/tools/devops/automation/templates/common/load_configuration.yml +++ b/tools/devops/automation/templates/common/load_configuration.yml @@ -75,8 +75,8 @@ steps: name: decisions displayName: 'Make decisions' - - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' + - pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index 91df3be177c1..ef1d67f3b689 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -17,30 +17,14 @@ steps: - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh displayName: 'Fix GitHub SSH host key' -- bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf - displayName: 'Clean workspace' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-bot.sh displayName: 'Clean bot' continueOnError: true timeoutInMinutes: 60 -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_python_env.ps1 - displayName: 'Show Python information' - -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/delete-library-dirs.sh - displayName: 'Delete library folders' - timeoutInMinutes: 5 - -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-results-dir.sh - workingDirectory: $(System.DefaultWorkingDirectory) - displayName: 'Clear results directory' - timeoutInMinutes: 5 - continueOnError: true - - bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh env: OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }} diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index 504c68d79372..fe366269b1a9 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -64,18 +64,16 @@ steps: condition: succeededOrFailed() # we do not care about the previous process cleanup continueOnError: true -- bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf - displayName: 'Clean workspace' - # download the packages that have been created, install them, later download the zip files that contain the already built # tests and execute them. -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' -- bash: | - ioreg -l | grep -e Manufacturer -e 'Vendor Name' - displayName: 'Dump Hardware' +- bash: $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])/tools/devops/automation/scripts/bash/clean-bot.sh + displayName: 'Clean bot' + continueOnError: true + timeoutInMinutes: 60 - bash: | if [[ $(ioreg -l | grep -e 'VMware' | wc -l) -ne 0 ]]; then @@ -95,11 +93,6 @@ steps: displayName: 'Set VM Vendor' -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-bot.sh - displayName: 'Clean bot' - continueOnError: true - timeoutInMinutes: 60 - # Use a cmdlet to check if the space available in the devices root system is larger than 50 gb. If there is not # enough space available it: # 1. Set the status of the build to error. It is not a failure since no tests have been ran. @@ -109,7 +102,7 @@ steps: Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 if ( -not (Test-HDFreeSpace -Size 5)) { - Set-Content -Path "$GITHUB_FAILURE_COMMENT_FILE" -Value "Not enough free space in the host." + Set-Content -Path "$GITHUB_FAILURE_COMMENT_FILE" -Value "Not enough free space in the host $Env:AGENT_MACHINENAME." exit 1 } env: diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index e8b12d92a345..31e71239c67f 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -99,7 +99,7 @@ steps: Import-Module ./MaciosCI.psd1 if ( -not (Test-HDFreeSpace -Size 20)) { - New-GitHubComment -Header "Tests failed catastrophically on $Env:CONTEXT" -Emoji ":fire:" -Description "Not enough free space in the host." + New-GitHubComment -Header "Tests failed catastrophically on $Env:CONTEXT" -Emoji ":fire:" -Description "Not enough free space in the host $Env:AGENT_MACHINENAME." Stop-Pipeline } env: diff --git a/tools/devops/automation/templates/tests/publish-html.yml b/tools/devops/automation/templates/tests/publish-html.yml index 79d44e48d23f..f87874c68f70 100644 --- a/tools/devops/automation/templates/tests/publish-html.yml +++ b/tools/devops/automation/templates/tests/publish-html.yml @@ -49,8 +49,8 @@ steps: - template: download-artifacts.yml -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' # build a message with all the content of all tests, to do so, we get the labels and to pass them to pwsh we do a join with ; # as the separator diff --git a/tools/devops/automation/templates/windows/build.yml b/tools/devops/automation/templates/windows/build.yml index c8c39389992f..80c95b91f2d2 100644 --- a/tools/devops/automation/templates/windows/build.yml +++ b/tools/devops/automation/templates/windows/build.yml @@ -37,8 +37,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Dump Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion'), contains(variables['Build.DefinitionName'], 'xamarin-macios-ci-tests'), contains(variables['Build.DefinitionName'], 'xamarin-macios-pr-tests')) }}: - download: macios @@ -127,9 +127,6 @@ steps: displayName: "Write and verify id_rsa" continueOnError: true -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\\xamarin-macios\\tools\\devops\\automation\\scripts\\MaciosCI.psd1 ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP pwd diff --git a/tools/devops/automation/templates/windows/reenable-mac.yml b/tools/devops/automation/templates/windows/reenable-mac.yml index 269802117011..bc9f0474b0a6 100644 --- a/tools/devops/automation/templates/windows/reenable-mac.yml +++ b/tools/devops/automation/templates/windows/reenable-mac.yml @@ -19,8 +19,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Dump Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - task: AzureKeyVault@2 inputs: diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 14ea0d05447a..7dc286ffff5a 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -33,8 +33,13 @@ steps: - checkout: maccore persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' + +- bash: $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])/tools/devops/automation/scripts/bash/clean-bot.sh + displayName: 'Clean bot' + continueOnError: true + timeoutInMinutes: 60 - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' From 6d263ab08c3b86d2000b2f56772f5440a00a7ed8 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 8 Oct 2024 01:22:07 +0200 Subject: [PATCH 47/59] Let's see how this goes. --- tests/dotnet/Windows/collect-binlogs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/dotnet/Windows/collect-binlogs.sh b/tests/dotnet/Windows/collect-binlogs.sh index 903097dbba82..5217786293a3 100755 --- a/tests/dotnet/Windows/collect-binlogs.sh +++ b/tests/dotnet/Windows/collect-binlogs.sh @@ -35,6 +35,9 @@ fi ps auxww > ~/remote_build_testing/processes.txt || true +sudo log collect --last 3h || true +zip -9r ~/remote_build_testing/windows-remote-logs.zip system_logs.logarchive || true + ls -la ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/ >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true cat ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/NuGet.config >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true cat ~/Library/Caches/Xamarin/XMA/SDKs/.home/.nuget/NuGet/NuGet.Config >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true From 7bc309058f98c0b04c4eca29e9ee2f6e01bf9d5f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 8 Oct 2024 11:13:24 +0200 Subject: [PATCH 48/59] Fixes --- tools/devops/automation/templates/common/setup.yml | 4 ++-- tools/devops/automation/templates/mac/build.yml | 2 +- tools/devops/automation/templates/windows/reserve-mac.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index ef1d67f3b689..8811f43cb03b 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -17,10 +17,10 @@ steps: - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh displayName: 'Fix GitHub SSH host key' -- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' displayName: 'Show Bot Info' -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-bot.sh +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' displayName: 'Clean bot' continueOnError: true timeoutInMinutes: 60 diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index fe366269b1a9..a2311ebeb463 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -70,7 +70,7 @@ steps: - pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' displayName: 'Show Bot Info' -- bash: $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])/tools/devops/automation/scripts/bash/clean-bot.sh +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' displayName: 'Clean bot' continueOnError: true timeoutInMinutes: 60 diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 7dc286ffff5a..8298a73c0da2 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -36,7 +36,7 @@ steps: - pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' displayName: 'Show Bot Info' -- bash: $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])/tools/devops/automation/scripts/bash/clean-bot.sh +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' displayName: 'Clean bot' continueOnError: true timeoutInMinutes: 60 From b23b609043230e6df8faecaecd184eece6e41770 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 8 Oct 2024 11:58:15 +0200 Subject: [PATCH 49/59] Bump --- msbuild/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index de02b1be47d2..a445a97ade22 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.57-pushpackage-diagnose-cancelled-posts] + [2.1.58-pushpackage-diagnose-cancelled-posts] [1.1.7] From d200e3fe6888fecab886e2c064b376cabdaeedcf Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 8 Oct 2024 15:17:21 +0200 Subject: [PATCH 50/59] More debug spew. --- msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs | 24 +++++++------- msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs | 18 +++++------ .../Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs | 31 +++++++++++++++++-- .../templates/windows/reserve-mac.yml | 2 +- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs index 5f75af19e2fe..5fd8f2c6c8e3 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; @@ -44,24 +45,20 @@ public class Ditto : XamarinTask, ITaskCallback, ICancelableTask { CancellationTokenSource? cancellationTokenSource; - void LogLine (string msg) - { - Log.LogMessage (MessageImportance.Low, msg); - Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); - Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); - } - + static int counter; public override bool Execute () { + var id = Interlocked.Increment (ref counter); + if (ShouldExecuteRemotely ()) { - LogLine ($"Ditto.Execute () will execute remotely"); + LogLine ($"{id} Ditto.Execute () will execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! }); - LogLine ($"Ditto.Execute () about to execute remotely"); + LogLine ($"{id} Ditto.Execute () about to execute remotely"); var rv = taskRunner.RunAsync (this).Result; - LogLine ($"Ditto.Execute () executed remotely: {rv}"); + LogLine ($"{id} Ditto.Execute () executed remotely: {rv}"); return rv; } @@ -87,11 +84,12 @@ public override bool Execute () } } - LogLine ($"Ditto.Execute () about to execute locally\n{Environment.StackTrace}"); + LogLine ($"{id} Ditto.Execute () about to execute locally. Args: {string.Join (" ", args)}\n{Environment.StackTrace}"); + var watch = Stopwatch.StartNew (); cancellationTokenSource = new CancellationTokenSource (); - cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this + cancellationTokenSource.CancelAfter (TimeSpan.FromSeconds (30)); // FIXME: remove this ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait (); - LogLine ($"Ditto.Execute () executed locally"); + LogLine ($"{id} Ditto.Execute () executed locally in {watch.Elapsed.TotalSeconds} seconds. Args: {string.Join (" ", args)}"); // Create a list of all the files we've copied var copiedFiles = new List (); diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs index 272861ffce24..c6d6ef4187ef 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/Unzip.cs @@ -39,31 +39,27 @@ public class Unzip : XamarinTask, ITaskCallback { [Output] public ITaskItem [] TouchedFiles { get; set; } = Array.Empty (); - void LogLine (string msg) - { - Log.LogMessage (MessageImportance.Low, msg); - Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}"); - Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}"); - } - CancellationTokenSource? cancellationTokenSource; + static int counter; public override bool Execute () { + var id = Interlocked.Increment (ref counter); + if (ShouldExecuteRemotely ()) { - LogLine ($"Unzip.Execute () about to execute remotely"); + LogLine ($"{id} Unzip.Execute () about to execute remotely"); var taskRunner = new TaskRunner (SessionId, BuildEngine4); var rv = taskRunner.RunAsync (this).Result; if (rv && CopyToWindows) CopyFilesToWindowsAsync (taskRunner, TouchedFiles).Wait (); - LogLine ($"Unzip.Execute () executed remotely: {rv}"); + LogLine ($"{id} Unzip.Execute () executed remotely: {rv}"); return rv; } - LogLine ($"Unzip.Execute () about to execute locally"); + LogLine ($"{id} Unzip.Execute () about to execute locally. ZipFilePath: {ZipFilePath} ExtractionPath: {ExtractionPath} Resource: {Resource}"); var rv2 = ExecuteLocally (); - LogLine ($"Unzip.Execute () executed locally"); + LogLine ($"{id} Unzip.Execute () executed locally"); return rv2; } diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs index ab5fc5e04ed0..6257ce620cb9 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs @@ -30,6 +30,19 @@ void VerifyTargetFrameworkMoniker () Log.LogError ($"The task {GetType ().Name} requires TargetFrameworkMoniker to be set."); } + static XamarinTask () + { + AppDomain.CurrentDomain.FirstChanceException += (object sender, System.Runtime.ExceptionServices.FirstChanceExceptionEventArgs fceea) => { + LogLine (null, "XamarinTask", $"FirstChanceException: {fceea.GetType ().FullName}\n" + + $"Exception.Message: {fceea.Exception.Message}\n" + + $"Exception.StackTrace: {fceea.Exception.StackTrace}\n" + + $"Exception.ToString (): {fceea.Exception}\n" + + $"Environment.StackTrace: {Environment.StackTrace}\n" + + $"FirstChanceException END"); + }; + LogLine (null, "XamarinTask", "Added FirstChanceException handler"); + } + public string Product { get { if (IsDotNet) @@ -121,6 +134,20 @@ protected System.Threading.Tasks.Task ExecuteAsync (string fileName, return ExecuteAsync (Log, fileName, arguments, sdkDevPath, environment, mergeOutput, showErrorIfFailure, workingDirectory); } + protected void LogLine (string msg) + { + LogLine (Log, GetType ().Name, msg); + } + + + protected static void LogLine (TaskLoggingHelper? log, string type, string msg) + { + log?.LogMessage (MessageImportance.Low, msg); + Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} {type} stdout: {msg}"); + Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} {type} stderr: {msg}"); + } + + internal protected static async System.Threading.Tasks.Task ExecuteAsync (TaskLoggingHelper log, string fileName, IList arguments, string? sdkDevPath = null, Dictionary? environment = null, bool mergeOutput = true, bool showErrorIfFailure = true, string? workingDirectory = null, CancellationToken? cancellationToken = null) { // Create a new dictionary if we're given one, to make sure we don't change the caller's dictionary. @@ -128,9 +155,9 @@ internal protected static async System.Threading.Tasks.Task ExecuteAs if (!string.IsNullOrEmpty (sdkDevPath)) launchEnvironment ["DEVELOPER_DIR"] = sdkDevPath; - log.LogMessage (MessageImportance.Normal, MSBStrings.M0001, fileName, StringUtils.FormatArguments (arguments)); + LogLine (log, "XamarinTask", string.Format (MSBStrings.M0001, fileName, StringUtils.FormatArguments (arguments))); var rv = await Execution.RunAsync (fileName, arguments, environment: launchEnvironment, mergeOutput: mergeOutput, workingDirectory: workingDirectory, cancellationToken: cancellationToken); - log.LogMessage (rv.ExitCode == 0 ? MessageImportance.Low : MessageImportance.High, MSBStrings.M0002, fileName, rv.ExitCode); + LogLine (log, "XamarinTask", string.Format (MSBStrings.M0002, fileName, rv.ExitCode)); // Show the output var output = rv.StandardOutput!.ToString (); diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 8298a73c0da2..83ab5a98fb4a 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -33,7 +33,7 @@ steps: - checkout: maccore persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail -- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' displayName: 'Show Bot Info' - bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' From 570b286f774ed69cec76ad71b277c3b55ffce9ec Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 8 Oct 2024 18:16:32 +0200 Subject: [PATCH 51/59] More debug spew. --- .../Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs index 6257ce620cb9..185ec1555abb 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Threading; using Microsoft.Build.Framework; @@ -140,13 +141,23 @@ protected void LogLine (string msg) } - protected static void LogLine (TaskLoggingHelper? log, string type, string msg) + protected static void LogLine (TaskLoggingHelper? log, string type, string? msg) { - log?.LogMessage (MessageImportance.Low, msg); + log?.LogMessage (MessageImportance.Low, msg ?? string.Empty); Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} {type} stdout: {msg}"); Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} {type} stderr: {msg}"); } + class CallbackWriter : TextWriter { + public Action? Callback; + public override void WriteLine (string? value) + { + if (value is not null) + Callback?.Invoke (value); + } + + public override Encoding Encoding => Encoding.UTF8; + } internal protected static async System.Threading.Tasks.Task ExecuteAsync (TaskLoggingHelper log, string fileName, IList arguments, string? sdkDevPath = null, Dictionary? environment = null, bool mergeOutput = true, bool showErrorIfFailure = true, string? workingDirectory = null, CancellationToken? cancellationToken = null) { @@ -155,16 +166,39 @@ internal protected static async System.Threading.Tasks.Task ExecuteAs if (!string.IsNullOrEmpty (sdkDevPath)) launchEnvironment ["DEVELOPER_DIR"] = sdkDevPath; + var execLogger = new CallbackWriter (); + execLogger.Callback = new Action (v => { + LogLine (log, "XamarinTask|log", v); + }); + var stdOutput = new StringBuilder (); + var stdOutputCallback = new Action ((v) => + { + LogLine (log, "XamarinTask|stdout", v); + if (v is not null) { + lock (stdOutput) + stdOutput.AppendLine (v); + } + }); + StringBuilder stdError = mergeOutput ? stdOutput : new StringBuilder (); + var stdErrorCallback = new Action ((v) => + { + LogLine (log, "XamarinTask|stderr", v); + if (v is not null) { + lock (stdError) + stdError.AppendLine (v); + } + }); + LogLine (log, "XamarinTask", string.Format (MSBStrings.M0001, fileName, StringUtils.FormatArguments (arguments))); - var rv = await Execution.RunAsync (fileName, arguments, environment: launchEnvironment, mergeOutput: mergeOutput, workingDirectory: workingDirectory, cancellationToken: cancellationToken); + var rv = await Execution.RunWithCallbacksAsync (fileName, arguments, environment: launchEnvironment, standardOutput: stdOutputCallback, standardError: stdErrorCallback, workingDirectory: workingDirectory, cancellationToken: cancellationToken, log: execLogger, timeout: TimeSpan.FromMinutes (1)); LogLine (log, "XamarinTask", string.Format (MSBStrings.M0002, fileName, rv.ExitCode)); // Show the output - var output = rv.StandardOutput!.ToString (); + var output = stdOutput.ToString ();// rv.StandardOutput!.ToString (); if (!mergeOutput) { if (output.Length > 0) output += Environment.NewLine; - output += rv.StandardError!.ToString (); + output += stdError.ToString (); // rv.StandardError!.ToString (); } if (output.Length > 0) { var importance = MessageImportance.Low; From 4091bf206ff37fc19ab62b1d3ff731e61b197952 Mon Sep 17 00:00:00 2001 From: GitHub Actions Autoformatter Date: Tue, 8 Oct 2024 16:19:41 +0000 Subject: [PATCH 52/59] Auto-format source code --- msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs index 185ec1555abb..94db2fdee3d7 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/XamarinTask.cs @@ -171,8 +171,7 @@ internal protected static async System.Threading.Tasks.Task ExecuteAs LogLine (log, "XamarinTask|log", v); }); var stdOutput = new StringBuilder (); - var stdOutputCallback = new Action ((v) => - { + var stdOutputCallback = new Action ((v) => { LogLine (log, "XamarinTask|stdout", v); if (v is not null) { lock (stdOutput) @@ -180,8 +179,7 @@ internal protected static async System.Threading.Tasks.Task ExecuteAs } }); StringBuilder stdError = mergeOutput ? stdOutput : new StringBuilder (); - var stdErrorCallback = new Action ((v) => - { + var stdErrorCallback = new Action ((v) => { LogLine (log, "XamarinTask|stderr", v); if (v is not null) { lock (stdError) From cc588f07b095c795fa2e636b83aec7e411a1f857 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 07:41:40 +0200 Subject: [PATCH 53/59] Bump --- msbuild/Directory.Build.props | 2 +- tools/common/Execution.cs | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/msbuild/Directory.Build.props b/msbuild/Directory.Build.props index a445a97ade22..034239166571 100644 --- a/msbuild/Directory.Build.props +++ b/msbuild/Directory.Build.props @@ -15,7 +15,7 @@ https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/ --> - [2.1.58-pushpackage-diagnose-cancelled-posts] + [2.1.59-pushpackage-diagnose-cancelled-posts] [1.1.7] diff --git a/tools/common/Execution.cs b/tools/common/Execution.cs index 8cc66b1c256b..424d739101fc 100644 --- a/tools/common/Execution.cs +++ b/tools/common/Execution.cs @@ -92,6 +92,26 @@ public Task RunAsync () StandardOutput ??= new StringWriter (); StandardError ??= new StringWriter (); + if (Log is not null) { + var ticker = new Thread (() => { + Log.WriteLine ($"Ticker for {p.StartInfo.FileName}"); + while (true) { + Thread.Sleep (TimeSpan.FromSeconds (1)); + if (tcs.Task.IsCompleted) { + Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: task completed with status '{tcs.Task.Status}'"); + break; + } else { + Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: task not complete yet (status = '{tcs.Task.Status}')"); + } + } + Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: Done"); + }) { + IsBackground = true, + Name = $"Thread ticker for {p.StartInfo.FileName}", + }; + ticker.Start (); + } + var thread = new Thread (() => { try { if (Log is not null) { From 91a3f057b6f596c260d61058aec6111013b9ef2f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 10:05:11 +0200 Subject: [PATCH 54/59] More debug spew. --- tools/common/Execution.cs | 53 +++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/tools/common/Execution.cs b/tools/common/Execution.cs index 424d739101fc..68d7fac38693 100644 --- a/tools/common/Execution.cs +++ b/tools/common/Execution.cs @@ -57,10 +57,12 @@ static Thread StartOutputThread (TaskCompletionSource tcs, object loc return thread; } + static int counter; public Task RunAsync () { var tcs = new TaskCompletionSource (); var lockobj = new object (); + var id = Interlocked.Increment (ref counter); try { var p = new Process (); @@ -92,26 +94,6 @@ public Task RunAsync () StandardOutput ??= new StringWriter (); StandardError ??= new StringWriter (); - if (Log is not null) { - var ticker = new Thread (() => { - Log.WriteLine ($"Ticker for {p.StartInfo.FileName}"); - while (true) { - Thread.Sleep (TimeSpan.FromSeconds (1)); - if (tcs.Task.IsCompleted) { - Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: task completed with status '{tcs.Task.Status}'"); - break; - } else { - Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: task not complete yet (status = '{tcs.Task.Status}')"); - } - } - Log.WriteLine ($"Ticker for {p.StartInfo.FileName}: Done"); - }) { - IsBackground = true, - Name = $"Thread ticker for {p.StartInfo.FileName}", - }; - ticker.Start (); - } - var thread = new Thread (() => { try { if (Log is not null) { @@ -120,9 +102,30 @@ public Task RunAsync () Log.WriteLine ("{0} {1}", p.StartInfo.FileName, p.StartInfo.Arguments); } + var watch = Stopwatch.StartNew (); p.Start (); var pid = p.Id; + if (Log is not null) { + var ticker = new Thread (() => { + Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid}"); + while (true) { + Thread.Sleep (TimeSpan.FromSeconds (1)); + if (tcs.Task.IsCompleted) { + Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid}: task completed with status '{tcs.Task.Status}'"); + break; + } else { + Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid}: task not complete yet (status = '{tcs.Task.Status}')"); + } + } + Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid}: Done"); + }) { + IsBackground = true, + Name = $"{id} Thread ticker for {p.StartInfo.FileName}", + }; + ticker.Start (); + } + var stdoutThread = StartOutputThread (tcs, lockobj, p.StandardOutput, StandardOutput, $"StandardOutput reader for {p.StartInfo.FileName} (PID: {pid})"); var stderrThread = StartOutputThread (tcs, lockobj, p.StandardError, StandardError, $"StandardError reader for {p.StartInfo.FileName} (PID: {pid})"); @@ -132,26 +135,28 @@ public Task RunAsync () p.Kill (); } catch (Exception ex) { // The process could be disposed already. Just ignore any exceptions here. - Log?.WriteLine ($"Failed to cancel and kill PID {pid}: {ex.Message}"); + Log?.WriteLine ($"{id} Failed to cancel and kill PID {pid}: {ex.Message}"); } }); if (Timeout.HasValue) { if (!p.WaitForExit ((int) Timeout.Value.TotalMilliseconds)) { - Log?.WriteLine ($"Command '{p.StartInfo.FileName} {p.StartInfo.Arguments}' didn't finish in {Timeout.Value.TotalMilliseconds} ms, and will be killed."); + Log?.WriteLine ($"{id} Command '{p.StartInfo.FileName} {p.StartInfo.Arguments}' didn't finish in {Timeout.Value.TotalMilliseconds} ms, and will be killed."); TimedOut = true; try { p.Kill (); } catch (Exception ex) { // According to the documentation, there can be exceptions here we can't prepare for, so just ignore them. - Log?.WriteLine ($"Failed to kill PID {pid}: {ex.Message}"); + Log?.WriteLine ($"{id} Failed to kill PID {pid}: {ex.Message}"); } } } // Always call this WaitForExit overload to be make sure the stdout/stderr buffers have been flushed, // even if we've called the WaitForExit (int) overload p.WaitForExit (); + watch.Stop (); ExitCode = p.ExitCode; + Log?.WriteLine ($"{id} Command '{p.StartInfo.FileName} {p.StartInfo.Arguments}' finished in {watch.Elapsed} with exit code {ExitCode}."); stdoutThread.Join (TimeSpan.FromSeconds (1)); stderrThread.Join (TimeSpan.FromSeconds (1)); @@ -164,7 +169,7 @@ public Task RunAsync () } }) { IsBackground = true, - Name = $"Thread waiting for {p.StartInfo.FileName} to finish", + Name = $"{id} Thread waiting for {p.StartInfo.FileName} to finish", }; thread.Start (); } catch (Exception e) { From cd682450064bd1a3d81ab1e602dfd0cdb6ee507e Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 10:07:50 +0200 Subject: [PATCH 55/59] Improve debug spew. --- tools/common/Execution.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/common/Execution.cs b/tools/common/Execution.cs index 68d7fac38693..ef6b7b5203ef 100644 --- a/tools/common/Execution.cs +++ b/tools/common/Execution.cs @@ -108,7 +108,7 @@ public Task RunAsync () if (Log is not null) { var ticker = new Thread (() => { - Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid}"); + Log.WriteLine ($"{id} Ticker for {p.StartInfo.FileName} with pid {pid} and timeout {(Timeout is null ? "" : Timeout.Value.ToString ())}"); while (true) { Thread.Sleep (TimeSpan.FromSeconds (1)); if (tcs.Task.IsCompleted) { From ad9ff2f5594e03cb25df35542685fba40e8cb333 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 12:29:47 +0200 Subject: [PATCH 56/59] Clean up after us. --- tests/dotnet/Windows/collect-binlogs.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/dotnet/Windows/collect-binlogs.sh b/tests/dotnet/Windows/collect-binlogs.sh index 5217786293a3..58f5ed337e0e 100755 --- a/tests/dotnet/Windows/collect-binlogs.sh +++ b/tests/dotnet/Windows/collect-binlogs.sh @@ -7,6 +7,15 @@ cd "$(dirname "${BASH_SOURCE[0]}")" TOPLEVEL="$(git rev-parse --show-toplevel)" +# Abort any agents that are still alive. +# Aborting creates a crash report, and we can investigate why they got stuck. +ps auxww || true +pkill -6 -f Broker.exe || true +pkill -6 -f Build.exe || true +pkill -6 -f Broker.dll || true +pkill -6 -f Build.dll || true +ps auxww || true + # Collect and zip up all the binlogs mkdir -p ~/remote_build_testing/binlogs rsync -avv --prune-empty-dirs --exclude 'artifacts/' --include '*/' --include '*.binlog' --exclude '*' "$TOPLEVEL/.." ~/remote_build_testing/binlogs @@ -38,6 +47,9 @@ ps auxww > ~/remote_build_testing/processes.txt || true sudo log collect --last 3h || true zip -9r ~/remote_build_testing/windows-remote-logs.zip system_logs.logarchive || true +# Collect any crash reports. +zip -9r ~/remote_build_testing/windows-remote-logs.zip ~/Library/Logs/DiagnosticReports || true + ls -la ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/ >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true cat ~/Library/Caches/Xamarin/XMA/SDKs/dotnet/NuGet.config >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true cat ~/Library/Caches/Xamarin/XMA/SDKs/.home/.nuget/NuGet/NuGet.Config >> ~/remote_build_testing/dotnet-debug.txt 2>&1 || true From cbce9f44b60ef2ad39d3ed930ce12e7e8cf31f37 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 13:26:14 +0200 Subject: [PATCH 57/59] Install arm64 dotnet. --- builds/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builds/Makefile b/builds/Makefile index 3a33d9754283..80f9374f0a2d 100644 --- a/builds/Makefile +++ b/builds/Makefile @@ -6,6 +6,8 @@ include $(TOP)/Make.config ifeq ($(shell arch),arm64) DOTNET_ARCH=arm64 +else ifeq ($(shell sysctl -n sysctl.proc_translated 2>/dev/null),1) +DOTNET_ARCH=arm64 else DOTNET_ARCH=x64 endif From a7ebd15006228ff027ff4345a8cca5a41d64ac5f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 13:26:23 +0200 Subject: [PATCH 58/59] Better cleanup. --- tools/devops/automation/scripts/clean-for-remote-tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/scripts/clean-for-remote-tests.sh b/tools/devops/automation/scripts/clean-for-remote-tests.sh index d05b9e0ec1a2..f796367149c4 100755 --- a/tools/devops/automation/scripts/clean-for-remote-tests.sh +++ b/tools/devops/automation/scripts/clean-for-remote-tests.sh @@ -18,6 +18,8 @@ rm -rf ~/remote_build_testing # Kill any existing brokers and builders ps auxww || true -pkill -f Broker.exe || true -pkill -f Build.exe || true +pkill -6 -f Broker.exe || true +pkill -6 -f Build.exe || true +pkill -6 -f Broker.dll || true +pkill -6 -f Build.dll || true ps auxww || true From 4479c120483586e7bc3231875bcaa17e0febd5b2 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 9 Oct 2024 13:27:38 +0200 Subject: [PATCH 59/59] tweaks --- builds/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builds/Makefile b/builds/Makefile index 80f9374f0a2d..31d213946add 100644 --- a/builds/Makefile +++ b/builds/Makefile @@ -72,7 +72,7 @@ downloads/%: downloads/%.nupkg $(Q) echo "Unzipped $*." downloads/$(DOTNET_INSTALL_NAME): dotnet-install.sh - $(Q) echo "Downloading and installing .NET $(DOTNET_VERSION) into $@..." + $(Q) echo "Downloading and installing .NET $(DOTNET_VERSION) ($(DOTNET_ARCH)) into $@..." $(Q) ./dotnet-install.sh --install-dir "$@.tmp" --version "$(DOTNET_VERSION)" --architecture $(DOTNET_ARCH) --no-path $$DOTNET_INSTALL_EXTRA_ARGS $(Q) rm -Rf "$@" $(Q) mv "$@.tmp" "$@"