From 43cd02dc2383f665e06172ea9e0519f2542aa5bc Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 10 Oct 2024 20:10:04 +0200 Subject: [PATCH 1/8] [msbuild] Improve the error message when the SupportedOSPlatformVersion is lower than the minimum. Fixes #21368. (#21369) This isn't very user friendly: ILLink : unknown error IL7000: An error occurred while executing the custom linker steps. Please review the build log for more information. ILLINK : error MT0073: Microsoft.iOS 18.0.8337 does not support a deployment target of 10.0 for iOS (the minimum is 11.0). Please select a newer deployment target in your project's Info.plist or change the SupportedOSPlatformVersion property in your project file. ILLINK : error MT2301: The linker step 'Setup' failed during processing: Microsoft.iOS 18.0.8337 does not support a deployment target of 10.0 for iOS (the minimum is 11.0). Please select a newer deployment target in your project's Info.plist or change the SupportedOSPlatformVersion property in your project file. [...]/packages/microsoft.net.illink.tasks/8.0.8/build/Microsoft.NET.ILLink.targets(87,5): error NETSDK1144: Optimizing assemblies for size failed. Optimization can be disabled by setting the PublishTrimmed property to false. So improve this to only show a single error message: The SupportedOSPlatformVersion value '10.0' in the project file is lower than the minimum value '11.0'. Fixes https://github.com/xamarin/xamarin-macios/issues/21368. --- dotnet/generate-target-platforms.csharp | 1 + .../MSBStrings.resx | 13 +++++++++ .../Tasks/CompileAppManifest.cs | 15 ++++++++++ msbuild/Xamarin.Shared/Xamarin.Shared.targets | 1 + tests/Makefile | 2 ++ tests/dotnet/UnitTests/ProjectTest.cs | 28 ++++++++++++++++++- .../TaskTests/CompileAppManifestTaskTests.cs | 16 +++-------- .../GeneratePlistTaskTests_Core.cs | 4 ++- 8 files changed, 66 insertions(+), 14 deletions(-) diff --git a/dotnet/generate-target-platforms.csharp b/dotnet/generate-target-platforms.csharp index b7a2ed4ad18e..bff196cab8f9 100755 --- a/dotnet/generate-target-platforms.csharp +++ b/dotnet/generate-target-platforms.csharp @@ -50,6 +50,7 @@ using (TextWriter writer = new StreamWriter (outputPath)) { writer.WriteLine ("\t"); writer.WriteLine ("\t"); writer.WriteLine ($"\t\t<{platform}MinSupportedOSPlatformVersion>{minSdkVersionString}"); + writer.WriteLine ($"\t\t$({platform}MinSupportedOSPlatformVersion)"); writer.WriteLine ("\t"); writer.WriteLine (""); } diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx index c25833c4c865..b68ef191e2a9 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx @@ -1563,6 +1563,19 @@ RuntimeIdentifier: don't translate (it's the name of a MSBuild property) + + Unable to parse the value '{0}' for the property '{1}. + + {0}: user-provided value + {1}: name of an MSBuild property + + + + + The SupportedOSPlatformVersion value '{0}' in the project file is lower than the minimum value '{1}'. + SupportedOSPlatformVersion: don't translate (it's the name of an MSBuild property) + + Adding reference to Xcode project output: '{0}'. The '%(CreateNativeReference)' metadata can be set to 'false' to opt out of this behavior. diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs index 3500599ef23a..7901c1f9f72d 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileAppManifest.cs @@ -62,6 +62,9 @@ public class CompileAppManifest : XamarinTask, ITaskCallback, ICancelableTask { public bool IsWatchExtension { get; set; } + [Required] + public string MinSupportedOSPlatformVersion { get; set; } = string.Empty; + public ITaskItem [] PartialAppManifests { get; set; } = Array.Empty (); [Required] @@ -302,6 +305,18 @@ bool SetMinimumOSVersion (PDictionary plist) minimumOSVersion = minimumOSVersionInManifest!; } + // Verify that the value is not lower than the minimum + if (!Version.TryParse (MinSupportedOSPlatformVersion, out var minSupportedVersion)) { + Log.LogError (MSBStrings.E7125 /* Unable to parse the value '{0}' for the property '{1}. */, MinSupportedOSPlatformVersion, "MinSupportedOSPlatformVersion"); + return false; + } else if (!Version.TryParse (SupportedOSPlatformVersion, out var supportedVersion)) { + Log.LogError (MSBStrings.E7125 /* Unable to parse the value '{0}' for the property '{1}. */, SupportedOSPlatformVersion, "SupportedOSPlatformVersion"); + return false; + } else if (minSupportedVersion > supportedVersion) { + Log.LogError (MSBStrings.E7126 /* The SupportedOSPlatformVersion value '{0}' in the project file is lower than the minimum value '{1}'." */, SupportedOSPlatformVersion, MinSupportedOSPlatformVersion); + return false; + } + // Write out our value plist [minimumVersionKey] = minimumOSVersion; diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.targets index 34adc65727dd..bc155c936ad5 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.targets @@ -570,6 +570,7 @@ Copyright (C) 2018 Microsoft. All rights reserved. IsWatchApp="$(IsWatchApp)" IsWatchExtension="$(IsWatchExtension)" IsXPCService="$(IsXPCService)" + MinSupportedOSPlatformVersion="$(MinSupportedOSPlatformVersion)" PartialAppManifests="@(PartialAppManifest)" ProjectDir="$(MSBuildProjectDirectory)" ResourcePrefix="$(_ResourcePrefix)" diff --git a/tests/Makefile b/tests/Makefile index 324448670a26..f13e4860ebe9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -87,6 +87,7 @@ test.config: Makefile $(TOP)/Make.config $(TOP)/mk/mono.mk $(TOP)/eng/Version.De @printf "ENABLE_XAMARIN=$(ENABLE_XAMARIN)\n" >> $@ @printf "XCODE_IS_STABLE=$(XCODE_IS_STABLE)\n" >> $@ @printf "XCODE_VERSION=$(XCODE_VERSION)\n" >> $@ + @printf "$(foreach platform,$(DOTNET_PLATFORMS_UPPERCASE),DOTNET_MIN_$(platform)_SDK_VERSION=$(DOTNET_MIN_$(platform)_SDK_VERSION)\\n)" | sed 's/^ //' >> $@ test-system.config: Makefile $(TOP)/Make.config $(TOP)/mk/mono.mk $(TOP)/eng/Version.Details.xml @rm -f $@ @@ -120,6 +121,7 @@ test-system.config: Makefile $(TOP)/Make.config $(TOP)/mk/mono.mk $(TOP)/eng/Ver @printf "ENABLE_XAMARIN=$(ENABLE_XAMARIN)\n" >> $@ @printf "XCODE_IS_STABLE=$(XCODE_IS_STABLE)\n" >> $@ @printf "XCODE_VERSION=$(XCODE_VERSION)\n" >> $@ + @printf "$(foreach platform,$(DOTNET_PLATFORMS_UPPERCASE),DOTNET_MIN_$(platform)_SDK_VERSION=$(DOTNET_MIN_$(platform)_SDK_VERSION)\\n)" | sed 's/^ //' >> $@ clean-local:: $(Q) $(SYSTEM_XBUILD) /t:Clean /p:Platform=iPhoneSimulator /p:Configuration=$(CONFIG) $(XBUILD_VERBOSITY) tests.sln diff --git a/tests/dotnet/UnitTests/ProjectTest.cs b/tests/dotnet/UnitTests/ProjectTest.cs index cc8541998b91..36e4a0dd8bd4 100644 --- a/tests/dotnet/UnitTests/ProjectTest.cs +++ b/tests/dotnet/UnitTests/ProjectTest.cs @@ -1861,7 +1861,7 @@ public void UnsupportedTargetPlatformVersion (ApplePlatform platform) // Pick a target platform version that we don't really support, // but don't show an error in .NET 8 because of backwards compat. // The earliest target OS version should do. - var minSupportedOSVersion = GetSupportedTargetPlatformVersions (platform).First (); + var minSupportedOSVersion = GetMinSupportedOSPlatformVersion (platform); var targetFrameworks = Configuration.DotNetTfm + "-" + platform.AsString ().ToLowerInvariant () + minSupportedOSVersion; var supportedApiVersions = GetSupportedApiVersions (platform, isCompat: false); @@ -1914,6 +1914,11 @@ string [] GetSupportedTargetPlatformVersions (ApplePlatform platform) .ToArray (); } + string GetMinSupportedOSPlatformVersion (ApplePlatform platform) + { + return Configuration.GetVariable ($"DOTNET_MIN_{platform.AsString ().ToUpperInvariant ()}_SDK_VERSION", "unknown MinSupportedOSPlatformVersion"); + } + [Test] [TestCase (ApplePlatform.MacCatalyst, "MtouchArch", "x86_64")] [TestCase (ApplePlatform.iOS, "MtouchArch", "ARMv7s")] @@ -2837,5 +2842,26 @@ static HashSet GetLinkedWithFrameworks (string path) } return rv; } + + [Test] + [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-x64", "13.1")] + [TestCase (ApplePlatform.iOS, "ios-arm64", "10.0")] + [TestCase (ApplePlatform.TVOS, "tvossimulator-x64", "10.0")] + [TestCase (ApplePlatform.MacOSX, "osx-arm64", "10.0")] + public void InvalidSupportedOSPlatformVersion (ApplePlatform platform, string runtimeIdentifiers, string version) + { + var project = "MySimpleApp"; + Configuration.IgnoreIfIgnoredPlatform (platform); + Configuration.AssertRuntimeIdentifiersAvailable (platform, runtimeIdentifiers); + + var minVersion = GetMinSupportedOSPlatformVersion (platform); + var project_path = GetProjectPath (project, runtimeIdentifiers: runtimeIdentifiers, platform: platform, out var appPath); + Clean (project_path); + var properties = GetDefaultProperties (runtimeIdentifiers); + properties ["SupportedOSPlatformVersion"] = version; + var rv = DotNet.AssertBuildFailure (project_path, properties); + var errors = BinLog.GetBuildLogErrors (rv.BinLogPath).ToArray (); + AssertErrorMessages (errors, $"The SupportedOSPlatformVersion value '{version}' in the project file is lower than the minimum value '{minVersion}'."); + } } } diff --git a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs index 00c3fa08f935..93cc0b5e6d41 100644 --- a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs +++ b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs @@ -22,24 +22,14 @@ CompileAppManifest CreateTask (string? tmpdir = null, ApplePlatform platform = A task.AppBundleName = "AppBundleName"; task.CompiledAppManifest = new TaskItem (Path.Combine (tmpdir, "TemporaryAppManifest.plist")); task.DefaultSdkVersion = Sdks.GetAppleSdk (platform).GetInstalledSdkVersions (false).First ().ToString ()!; + task.MinSupportedOSPlatformVersion = "10.0"; + task.SupportedOSPlatformVersion = "15.0"; task.SdkVersion = task.DefaultSdkVersion ?? string.Empty; task.TargetFrameworkMoniker = TargetFramework.GetTargetFramework (platform, true).ToString (); return task; } - [Test] - public void DefaultMinimumOSVersion () - { - var dir = Cache.CreateTemporaryDirectory (); - var task = CreateTask (dir); - - ExecuteTask (task); - - var plist = PDictionary.FromFile (task.CompiledAppManifest!.ItemSpec)!; - Assert.AreEqual (task.SdkVersion, plist.GetMinimumOSVersion (), "MinimumOSVersion"); - } - [Test] public void MainMinimumOSVersions () { @@ -52,6 +42,7 @@ public void MainMinimumOSVersions () main.Save (mainPath); task.AppManifest = new TaskItem (mainPath); + task.SupportedOSPlatformVersion = "14.0"; ExecuteTask (task); @@ -78,6 +69,7 @@ public void MultipleMinimumOSVersions () task.AppManifest = new TaskItem (mainPath); task.PartialAppManifests = new [] { new TaskItem (partialPath) }; + task.SupportedOSPlatformVersion = "14.0"; ExecuteTask (task); diff --git a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/GeneratePlistTaskTests/GeneratePlistTaskTests_Core.cs b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/GeneratePlistTaskTests/GeneratePlistTaskTests_Core.cs index b82d27061176..85487ed1fb8c 100644 --- a/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/GeneratePlistTaskTests/GeneratePlistTaskTests_Core.cs +++ b/tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/GeneratePlistTaskTests/GeneratePlistTaskTests_Core.cs @@ -50,6 +50,8 @@ protected virtual void ConfigureTask (bool isDotNet) Task.CompiledAppManifest = new TaskItem (Path.Combine (Cache.CreateTemporaryDirectory (), "AppBundlePath", "Info.plist")); Task.AssemblyName = assemblyName; Task.AppManifest = new TaskItem (CreateTempFile ("foo.plist")); + Task.MinSupportedOSPlatformVersion = "10.0"; + Task.SupportedOSPlatformVersion = "15.0"; Task.SdkVersion = "10.0"; Plist = new PDictionary (); @@ -70,7 +72,7 @@ public override void Setup () ConfigureTask (IsDotNet); - Task.Execute (); + ExecuteTask (Task); CompiledPlist = PDictionary.FromFile (Task.CompiledAppManifest.ItemSpec); } From 46ed02ba8c47cf43169f42776ab4d2f64e28b70d Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 08:36:29 +0200 Subject: [PATCH 2/8] [tests] Ignore a few tests that fail on iOS Simulator/arm64. (#21403) Ref: https://github.com/xamarin/xamarin-macios/issues/19781 --- tests/monotouch-test/ObjCRuntime/RegistrarTestGenerated.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/monotouch-test/ObjCRuntime/RegistrarTestGenerated.cs b/tests/monotouch-test/ObjCRuntime/RegistrarTestGenerated.cs index 2a500d50b312..b2787b3199ae 100644 --- a/tests/monotouch-test/ObjCRuntime/RegistrarTestGenerated.cs +++ b/tests/monotouch-test/ObjCRuntime/RegistrarTestGenerated.cs @@ -9,7 +9,7 @@ public partial class RegistrarTestGenerated { void AssertIfIgnored ([CallerMemberName] string testCase = null) { switch (testCase) { -#if __MACCATALYST__ +#if __MACCATALYST__ || __IOS__ case "NSNumberBindAs_Boolean_Array_Overrides": case "NSNumberBindAs_Byte_Array_Overrides": case "NSNumberBindAs_Double_Array_Overrides": @@ -25,7 +25,11 @@ void AssertIfIgnored ([CallerMemberName] string testCase = null) case "NSNumberBindAs_UInt32_Array_Overrides": case "NSNumberBindAs_UInt64_Array_Overrides": // https://github.com/xamarin/xamarin-macios/issues/19781 +#if __MACCATALYST__ if (Runtime.IsARM64CallingConvention) +#elif __IOS__ + if (Runtime.IsARM64CallingConvention && Runtime.Arch == Arch.SIMULATOR) +#endif Assert.Ignore ("https://github.com/xamarin/xamarin-macios/issues/19781"); break; #endif From 22cb6e8cb24d0c1b524a19da7ff4fee324f4e5ca Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 08:37:41 +0200 Subject: [PATCH 3/8] [devops] Remove the wget dependency. (#21405) It doesn't look like it's used anywhere (and I don't have it on my system either). --- tools/devops/provision-shared.in.csx | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/devops/provision-shared.in.csx b/tools/devops/provision-shared.in.csx index bd0989fc68d8..5ad2d0b38c50 100644 --- a/tools/devops/provision-shared.in.csx +++ b/tools/devops/provision-shared.in.csx @@ -72,7 +72,6 @@ void ProvisionBrewPackages () "yamllint", "p7zip", "msitools", - "wget", "azure-cli" ); } From 657f0169c53ad9362bfede90e4a1537771069b0f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 09:32:15 +0200 Subject: [PATCH 4/8] [tools] Ignore any interpreter settings when the current runtime isn't MonoVM. Fixes #20398. (#21406) Tests that exercise this code path will be included in a different PR. Fixes https://github.com/xamarin/xamarin-macios/issues/20398. --- tools/common/Application.cs | 6 ++++++ tools/dotnet-linker/LinkerConfiguration.cs | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/tools/common/Application.cs b/tools/common/Application.cs index 6e571717d6cf..b82a3e85cca8 100644 --- a/tools/common/Application.cs +++ b/tools/common/Application.cs @@ -546,6 +546,12 @@ public void ParseInterpreter (string value) InterpretedAssemblies.AddRange (value.Split (new char [] { ',' }, StringSplitOptions.RemoveEmptyEntries)); } + public void UnsetInterpreter () + { + UseInterpreter = false; + InterpretedAssemblies.Clear (); + } + #if !NET public void ParseI18nAssemblies (string i18n) { diff --git a/tools/dotnet-linker/LinkerConfiguration.cs b/tools/dotnet-linker/LinkerConfiguration.cs index 78e29d716f07..e450d03902ea 100644 --- a/tools/dotnet-linker/LinkerConfiguration.cs +++ b/tools/dotnet-linker/LinkerConfiguration.cs @@ -425,6 +425,11 @@ public static LinkerConfiguration GetInstance (LinkContext context) if (Driver.TargetFramework.Platform != Platform) throw ErrorHelper.CreateError (99, "Inconsistent platforms. TargetFramework={0}, Platform={1}", Driver.TargetFramework.Platform, Platform); + if (Application.XamarinRuntime != XamarinRuntime.MonoVM && Application.UseInterpreter) { + Driver.Log (4, "The interpreter is enabled, but the current runtime isn't MonoVM. The interpreter settings will be ignored."); + Application.UnsetInterpreter (); + } + Driver.ValidateXcode (Application, false, false); Application.InitializeCommon (); From 2f7c126ac9d6516135100d69063faa86717cfbbb Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 12:13:29 +0200 Subject: [PATCH 5/8] [dotnet] Fix building universal apps using NativeAOT when the default RuntimeIdentifiers value is set. Fixes #19391. (#21412) We need to set "UseCurrentRuntimeIdentifier=false", we must only set it for the outer build for universal builds - when `RuntimeIdentifiers` is set - but that means we can only do it *after* we set any default value for `RuntimeIdentifiers`. Fixes https://github.com/xamarin/xamarin-macios/issues/19391#issuecomment-2405499973. --- dotnet/targets/Xamarin.Shared.Sdk.props | 7 +++---- tests/dotnet/UnitTests/ProjectTest.cs | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/targets/Xamarin.Shared.Sdk.props b/dotnet/targets/Xamarin.Shared.Sdk.props index b628bc55eec1..cdea9479e7fd 100644 --- a/dotnet/targets/Xamarin.Shared.Sdk.props +++ b/dotnet/targets/Xamarin.Shared.Sdk.props @@ -94,10 +94,6 @@ true true - - - false - @@ -142,6 +138,9 @@ + + false + $(CustomAfterDirectoryBuildTargets);$(MSBuildThisFileDirectory)Xamarin.Shared.Sdk.Trimming.props diff --git a/tests/dotnet/UnitTests/ProjectTest.cs b/tests/dotnet/UnitTests/ProjectTest.cs index 36e4a0dd8bd4..5e881d167ffc 100644 --- a/tests/dotnet/UnitTests/ProjectTest.cs +++ b/tests/dotnet/UnitTests/ProjectTest.cs @@ -1744,6 +1744,7 @@ public void BuildMyNativeAotAppWithTrimAnalysisWarning (ApplePlatform platform, [TestCase (ApplePlatform.MacOSX, "osx-x64", "Release")] [TestCase (ApplePlatform.MacOSX, "osx-arm64;osx-x64", "Debug")] [TestCase (ApplePlatform.MacOSX, "osx-arm64;osx-x64", "Release")] + [TestCase (ApplePlatform.MacOSX, "", "Release")] [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-arm64", "Debug")] [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-arm64", "Release")] [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-arm64;maccatalyst-x64", "Debug")] From 48f559182931d4633895cd3b9dc7f0eccc253d89 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 13:26:45 +0200 Subject: [PATCH 6/8] [devops] Improved diagnostics and implement deadlocked process termination. (#21317) * Unify the code to collect diagnostic information about a bot. * Unify some of the cleanup code to prepare a bot as well. * Implement code to terminate processes on a bot that's used more than a day of CPU (presumably these processes are stuck for some reason). --- .../automation/scripts/bash/clean-bot.sh | 20 ++++++++++++- .../scripts/bash/clean-results-dir.sh | 4 --- .../scripts/bash/clean-simulator-runtime.sh | 5 ++++ .../scripts/bash/delete-library-dirs.sh | 6 ---- .../scripts/bash/kill-deadlocked-processes.sh | 27 +++++++++++++++++ .../run-generator-tests-on-windows.ps1 | 2 +- .../{show_env.ps1 => show_bot_info.ps1} | 29 ++++++++++++++----- .../automation/scripts/show_python_env.ps1 | 6 ---- .../build/api-diff-process-results.yml | 4 +-- .../automation/templates/common/configure.yml | 4 +-- .../templates/common/load_configuration.yml | 4 +-- .../automation/templates/common/setup.yml | 22 ++------------ .../devops/automation/templates/mac/build.yml | 21 +++++--------- .../automation/templates/tests/build.yml | 2 +- .../templates/tests/publish-html.yml | 4 +-- .../automation/templates/windows/build.yml | 7 ++--- .../templates/windows/reenable-mac.yml | 4 +-- .../templates/windows/reserve-mac.yml | 8 +++++ 18 files changed, 105 insertions(+), 74 deletions(-) delete mode 100755 tools/devops/automation/scripts/bash/clean-results-dir.sh delete mode 100755 tools/devops/automation/scripts/bash/delete-library-dirs.sh create mode 100755 tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh rename tools/devops/automation/scripts/{show_env.ps1 => show_bot_info.ps1} (51%) delete mode 100644 tools/devops/automation/scripts/show_python_env.ps1 diff --git a/tools/devops/automation/scripts/bash/clean-bot.sh b/tools/devops/automation/scripts/bash/clean-bot.sh index 923d353c1a8d..c48e4d5895f7 100755 --- a/tools/devops/automation/scripts/bash/clean-bot.sh +++ b/tools/devops/automation/scripts/bash/clean-bot.sh @@ -12,6 +12,15 @@ df -h # We don't care about errors in this section, we just want to clean as much as possible set +e +# Clean workspace +( + REPO_PATH="SYSTEM_DEFAULTWORKINGDIRECTORY/$(basename "$BUILD_REPOSITORY_NAME")" + if test -d "$REPO_PATH"; then + cd "$REPO_PATH" + git clean -xfd + fi +) + # Delete all the simulator devices. These can take up a lot of space over time (I've seen 100+GB on the bots) /Applications/Xcode.app/Contents/Developer/usr/bin/simctl delete all @@ -118,7 +127,9 @@ XCODE_SELECT=$(xcode-select -p) for oldXcode in "${oldXcodes[@]}"; do if [ "$XCODE_SELECT" != "$oldXcode/Contents/Developer" ]; then - sudo rm -Rf "$oldXcode" + if test -d "$oldXcode"; then + sudo rm -Rf "$oldXcode" + fi else echo "Not removing $oldXcode because is the currently selected one." fi @@ -126,6 +137,13 @@ done DIR="$(dirname "${BASH_SOURCE[0]}")" "$DIR"/clean-simulator-runtime.sh +"$DIR"/kill-deadlocked-processes.sh + +# Remove legacy Xamarin/MonoTouch stuff +sudo rm -Rf /Developer/MonoTouch +sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework +sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework +ls -R /Library/Frameworks # Print disk status after cleaning df -h diff --git a/tools/devops/automation/scripts/bash/clean-results-dir.sh b/tools/devops/automation/scripts/bash/clean-results-dir.sh deleted file mode 100755 index 3e1aa2dfb4e1..000000000000 --- a/tools/devops/automation/scripts/bash/clean-results-dir.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -ex - -rm -Rvf package -time make -C xamarin-macios/ git-clean-all diff --git a/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh b/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh index ed23cf1193c2..dfddf64f38d9 100755 --- a/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh +++ b/tools/devops/automation/scripts/bash/clean-simulator-runtime.sh @@ -5,6 +5,11 @@ set -o pipefail IFS=$'\n\t' +# delete all watchOS simulators, we don't need them anymore +for i in $(xcrun simctl runtime list | grep "watchOS.*Ready" | sed -e 's/.* - //' -e 's/ .*//'); do + xcrun simctl runtime delete "$i" +done + xcrun simctl runtime list -j > simruntime.json cat simruntime.json diff --git a/tools/devops/automation/scripts/bash/delete-library-dirs.sh b/tools/devops/automation/scripts/bash/delete-library-dirs.sh deleted file mode 100755 index 3c647824f72b..000000000000 --- a/tools/devops/automation/scripts/bash/delete-library-dirs.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -ex - -sudo rm -Rf /Developer/MonoTouch -sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework -sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework -ls -R /Library/Frameworks diff --git a/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh b/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh new file mode 100755 index 000000000000..7610d51a7200 --- /dev/null +++ b/tools/devops/automation/scripts/bash/kill-deadlocked-processes.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +echo "Looking for processes that have been stuck for more than a day, and will try to kill them." + +# Collect the list of processes for the current user, including the CPU time. +# We then split the CPU time into separate fields, so that it's easier to figure out the total number of minutes later on. +IFS=$'\n' +PROCESSES=() +while IFS='' read -r line; do PROCESSES+=("$line"); done < <(ps -o cputime=,pid=,user=,lstart=,args= -U "$USER" -w -w -w | sed -e 's/\([0-9]*\):\([0-9][0-9]\)\.\([0-9][0-9]\)/\1 m \2 s \3 ms/' | sort -nr) + +IFS=' ' +for process in "${PROCESSES[@]}"; do + IFS=" " read -r -a FIELDS <<< "$process" + minutes=${FIELDS[0]} + pid=${FIELDS[6]} + + echo "$process" + + # looking for processes that have spent more than a day using the CPU (24h * 60min = 1440min) + if (( minutes > 1440 )); then + echo " This process has been stuck for more than $minutes minutes, so assuming it's deadlocked and we'll try to kill it:" + echo " kill -9 $pid" + kill -9 "$pid" | sed 's/^/ /' || true + fi +done + +echo "No (more) processes stuck for more than a day." diff --git a/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 b/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 index 493b3026c0c2..a59e58043535 100644 --- a/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 +++ b/tools/devops/automation/scripts/run-generator-tests-on-windows.ps1 @@ -1,5 +1,5 @@ # Dump the environment to see what we're working with. -& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\show_env.ps1" +& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\$($Env:BUILD_REPOSITORY_NAME.Split('/')[1])\tools\devops\automation\scripts\show_bot_info.ps1" # Set a few variables $Env:DOTNET = "$Env:BUILD_SOURCESDIRECTORY\xamarin-macios\tests\dotnet\Windows\bin\dotnet\dotnet.exe" diff --git a/tools/devops/automation/scripts/show_env.ps1 b/tools/devops/automation/scripts/show_bot_info.ps1 similarity index 51% rename from tools/devops/automation/scripts/show_env.ps1 rename to tools/devops/automation/scripts/show_bot_info.ps1 index 7852d1ad7c7a..0369422be80e 100644 --- a/tools/devops/automation/scripts/show_env.ps1 +++ b/tools/devops/automation/scripts/show_bot_info.ps1 @@ -8,13 +8,14 @@ if ($IsMacOS -or $IsLinux) { Write-Host "COMPUTERNAME: ${env:COMPUTERNAME}" } -gci env: | format-table -autosize - -gci env: | format-table -autosize | Out-String -Width 8192 - -gci env: | format-table -autosize -wrap +Get-ChildItem env: | Sort-Object -Property Name | Format-Table -AutoSize | Out-String -Width 81920 if ($IsMacOS) { + Write-Host "" + Write-Host "## Uptime" + Write-Host "" + uptime + Write-Host "" Write-Host "## System profile" Write-Host "" @@ -25,10 +26,24 @@ if ($IsMacOS) { Write-Host "" ifconfig | grep 'inet ' + Write-Host "" + Write-Host "## Top processes (ps)" + Write-Host "" + ps aux + + Write-Host "" + Write-Host "## Python3 location:" + Write-Host "" + which python3 + + Write-Host "" + Write-Host "## Pip3 version:" + Write-Host "" + pip3 -V Write-Host "" - Write-Host "## Top processes" + Write-Host "## Hardware info" Write-Host "" - top -l 1 -o TIME + ioreg -l | grep -e Manufacturer -e 'Vendor Name' } diff --git a/tools/devops/automation/scripts/show_python_env.ps1 b/tools/devops/automation/scripts/show_python_env.ps1 deleted file mode 100644 index 79873375cf32..000000000000 --- a/tools/devops/automation/scripts/show_python_env.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -Write-Host "Python3 location" -which python3 - -Write-Host "Pip3 version" -pip3 -V - diff --git a/tools/devops/automation/templates/build/api-diff-process-results.yml b/tools/devops/automation/templates/build/api-diff-process-results.yml index c8b3bf3cc8bb..4d3f74c10c27 100644 --- a/tools/devops/automation/templates/build/api-diff-process-results.yml +++ b/tools/devops/automation/templates/build/api-diff-process-results.yml @@ -27,8 +27,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} - - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' + - pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) { diff --git a/tools/devops/automation/templates/common/configure.yml b/tools/devops/automation/templates/common/configure.yml index e909e535fe88..c30328ef600c 100644 --- a/tools/devops/automation/templates/common/configure.yml +++ b/tools/devops/automation/templates/common/configure.yml @@ -65,8 +65,8 @@ steps: name: decisions displayName: 'Make decisions' -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 diff --git a/tools/devops/automation/templates/common/load_configuration.yml b/tools/devops/automation/templates/common/load_configuration.yml index 5ecac652b11f..536cad18268f 100644 --- a/tools/devops/automation/templates/common/load_configuration.yml +++ b/tools/devops/automation/templates/common/load_configuration.yml @@ -75,8 +75,8 @@ steps: name: decisions displayName: 'Make decisions' - - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' + - pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index 91df3be177c1..8811f43cb03b 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -17,30 +17,14 @@ steps: - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh displayName: 'Fix GitHub SSH host key' -- bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf - displayName: 'Clean workspace' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-bot.sh +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' displayName: 'Clean bot' continueOnError: true timeoutInMinutes: 60 -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_python_env.ps1 - displayName: 'Show Python information' - -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/delete-library-dirs.sh - displayName: 'Delete library folders' - timeoutInMinutes: 5 - -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-results-dir.sh - workingDirectory: $(System.DefaultWorkingDirectory) - displayName: 'Clear results directory' - timeoutInMinutes: 5 - continueOnError: true - - bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh env: OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }} diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index 504c68d79372..fca31bfd84fd 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -64,18 +64,16 @@ steps: condition: succeededOrFailed() # we do not care about the previous process cleanup continueOnError: true -- bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf - displayName: 'Clean workspace' - # download the packages that have been created, install them, later download the zip files that contain the already built # tests and execute them. -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' -- bash: | - ioreg -l | grep -e Manufacturer -e 'Vendor Name' - displayName: 'Dump Hardware' +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' + displayName: 'Clean bot' + continueOnError: true + timeoutInMinutes: 60 - bash: | if [[ $(ioreg -l | grep -e 'VMware' | wc -l) -ne 0 ]]; then @@ -95,11 +93,6 @@ steps: displayName: 'Set VM Vendor' -- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-bot.sh - displayName: 'Clean bot' - continueOnError: true - timeoutInMinutes: 60 - # Use a cmdlet to check if the space available in the devices root system is larger than 50 gb. If there is not # enough space available it: # 1. Set the status of the build to error. It is not a failure since no tests have been ran. @@ -109,7 +102,7 @@ steps: Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 if ( -not (Test-HDFreeSpace -Size 5)) { - Set-Content -Path "$GITHUB_FAILURE_COMMENT_FILE" -Value "Not enough free space in the host." + Set-Content -Path "$GITHUB_FAILURE_COMMENT_FILE" -Value "Not enough free space in the host $Env:AGENT_MACHINENAME." exit 1 } env: diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index 58396ef89bd5..98c2482478b9 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -99,7 +99,7 @@ steps: Import-Module ./MaciosCI.psd1 if ( -not (Test-HDFreeSpace -Size 20)) { - New-GitHubComment -Header "Tests failed catastrophically on $Env:CONTEXT" -Emoji ":fire:" -Description "Not enough free space in the host." + New-GitHubComment -Header "Tests failed catastrophically on $Env:CONTEXT" -Emoji ":fire:" -Description "Not enough free space in the host $Env:AGENT_MACHINENAME." Stop-Pipeline } env: diff --git a/tools/devops/automation/templates/tests/publish-html.yml b/tools/devops/automation/templates/tests/publish-html.yml index 79d44e48d23f..e163ac2ba609 100644 --- a/tools/devops/automation/templates/tests/publish-html.yml +++ b/tools/devops/automation/templates/tests/publish-html.yml @@ -49,8 +49,8 @@ steps: - template: download-artifacts.yml -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' # build a message with all the content of all tests, to do so, we get the labels and to pass them to pwsh we do a join with ; # as the separator diff --git a/tools/devops/automation/templates/windows/build.yml b/tools/devops/automation/templates/windows/build.yml index c8c39389992f..8f5545baee49 100644 --- a/tools/devops/automation/templates/windows/build.yml +++ b/tools/devops/automation/templates/windows/build.yml @@ -37,8 +37,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Dump Environment' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion'), contains(variables['Build.DefinitionName'], 'xamarin-macios-ci-tests'), contains(variables['Build.DefinitionName'], 'xamarin-macios-pr-tests')) }}: - download: macios @@ -127,9 +127,6 @@ steps: displayName: "Write and verify id_rsa" continueOnError: true -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\\xamarin-macios\\tools\\devops\\automation\\scripts\\MaciosCI.psd1 ssh -v -i "$(ID_RSA_PATH)" -o IdentitiesOnly=yes -o StrictHostKeyChecking=no builder@$Env:MAC_AGENT_IP pwd diff --git a/tools/devops/automation/templates/windows/reenable-mac.yml b/tools/devops/automation/templates/windows/reenable-mac.yml index 269802117011..833788b3092b 100644 --- a/tools/devops/automation/templates/windows/reenable-mac.yml +++ b/tools/devops/automation/templates/windows/reenable-mac.yml @@ -19,8 +19,8 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Dump Environment' +- pwsh: '& "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' - task: AzureKeyVault@2 inputs: diff --git a/tools/devops/automation/templates/windows/reserve-mac.yml b/tools/devops/automation/templates/windows/reserve-mac.yml index 9c8710d9b619..8298a73c0da2 100644 --- a/tools/devops/automation/templates/windows/reserve-mac.yml +++ b/tools/devops/automation/templates/windows/reserve-mac.yml @@ -33,6 +33,14 @@ steps: - checkout: maccore persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail +- pwsh: '"$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/$($Env:BUILD_REPOSITORY_NAME.Split(''/'')[1])/tools/devops/automation/scripts/show_bot_info.ps1"' + displayName: 'Show Bot Info' + +- bash: '$SYSTEM_DEFAULTWORKINGDIRECTORY/${BUILD_REPOSITORY_NAME/#*\//}/tools/devops/automation/scripts/bash/clean-bot.sh' + displayName: 'Clean bot' + continueOnError: true + timeoutInMinutes: 60 + - bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' name: disableCodeQLOnArm64 From 0482be8bd331d63c9844031c975334e59ab0ef35 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 11 Oct 2024 16:34:16 +0200 Subject: [PATCH 7/8] [devops] Remove the msitools dependency. (#21415) According to @pjcollins the msi files we were generated aren't used anywhere, so this should be safe. --- dotnet/Makefile | 27 +-------------------------- tools/devops/provision-shared.in.csx | 1 - 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/dotnet/Makefile b/dotnet/Makefile index 7e1eb766c266..a67f5289fe37 100644 --- a/dotnet/Makefile +++ b/dotnet/Makefile @@ -436,17 +436,6 @@ WINDOWS_PACKAGE_TARGETS += $(DOTNET_PKG_DIR)/Microsoft.$1.Windows.Bundle.$2.zip endef $(foreach platform,$(DOTNET_WINDOWS_PLATFORMS),$(eval $(call CreateWindowsBundle,$(platform),$($(platform)_NUGET_VERSION_NO_METADATA),$(shell echo $(platform) | tr A-Z a-z),$(shell echo $(platform) | tr a-z A-Z)))) -define CreateMsi -$(TMP_PKG_DIR)/Microsoft.NET.Workload.$1.$2.wsx: ./generate-wix.csharp Makefile $(TMP_PKG_DIR)/Microsoft.$1.Windows.Bundle.$2.zip - $$(Q_GEN) ./generate-wix.csharp "$1" "$$@" "$(TMP_PKG_DIR)/Microsoft.$1.Windows.Bundle.$2.zip.tmpdir/dotnet" "$2" - -$(TMP_PKG_DIR)/Microsoft.NET.Workload.$1.$2.msi: $(TMP_PKG_DIR)/Microsoft.NET.Workload.$1.$2.wsx .stamp-check-wixl - $$(Q_GEN) wixl -o "$$@" "$$<" -a x64 - -MSI_TARGETS += $(DOTNET_PKG_DIR)/Microsoft.NET.Workload.$1.$2.msi -endef -$(foreach platform,$(DOTNET_WINDOWS_PLATFORMS),$(eval $(call CreateMsi,$(platform),$($(platform)_NUGET_VERSION_NO_METADATA)))) - NUGET_SOURCES:=$(shell grep https://pkgs.dev.azure.com $(TOP)/NuGet.config | sed -e 's/.*value="//' -e 's/".*//') .stamp-install-workloads: Makefile $(WORKLOAD_TARGETS) $(RUNTIME_PACKS) $(REF_PACKS) $(SDK_PACKS) $(TEMPLATE_PACKS) $(WORKLOAD_PACKS) $(Q) $(DOTNET) workload install --skip-manifest-update \ @@ -457,23 +446,9 @@ NUGET_SOURCES:=$(shell grep https://pkgs.dev.azure.com $(TOP)/NuGet.config | sed TARGETS += .stamp-install-workloads -.stamp-check-wixl: - $(Q) if ! type wixl; then \ - echo "Installing msitools to get wixl..."; \ - if ! brew install msitools; then \ - if ! type wixl; then \ - echo "Failed to install wixl"; \ - exit 1; \ - fi; \ - fi; \ - echo "Installed msitools"; \ - fi - $(Q) touch $@ - TARGETS += $(WORKLOAD_TARGETS) $(WINDOWS_PACKAGE_TARGETS) -msi: $(MSI_TARGETS) -package: $(PACKAGE_TARGETS) $(MSI_TARGETS) $(WINDOWS_PACKAGE_TARGETS) +package: $(PACKAGE_TARGETS) $(WINDOWS_PACKAGE_TARGETS) # Helper targets for templates # diff --git a/tools/devops/provision-shared.in.csx b/tools/devops/provision-shared.in.csx index 5ad2d0b38c50..23947808f1aa 100644 --- a/tools/devops/provision-shared.in.csx +++ b/tools/devops/provision-shared.in.csx @@ -71,7 +71,6 @@ void ProvisionBrewPackages () "shellcheck", "yamllint", "p7zip", - "msitools", "azure-cli" ); } From 57380a58b8c123d5e3dfeb23dbe0eacb72b49bc7 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 13 Oct 2024 17:17:29 -0400 Subject: [PATCH 8/8] [CI] Fix the download step (#21398) Allow to download the artifacts from the build that triggered the pipeline. --- .../automation/run-post-ci-build-api-scan.yml | 1 - .../templates/governance/apiscan.yml | 33 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tools/devops/automation/run-post-ci-build-api-scan.yml b/tools/devops/automation/run-post-ci-build-api-scan.yml index bddf49deed90..c95684af0b41 100644 --- a/tools/devops/automation/run-post-ci-build-api-scan.yml +++ b/tools/devops/automation/run-post-ci-build-api-scan.yml @@ -15,4 +15,3 @@ extends: template: templates/pipelines/run-api-scan.yml parameters: isPR: false - diff --git a/tools/devops/automation/templates/governance/apiscan.yml b/tools/devops/automation/templates/governance/apiscan.yml index c68489fc2152..b154cd6aed62 100644 --- a/tools/devops/automation/templates/governance/apiscan.yml +++ b/tools/devops/automation/templates/governance/apiscan.yml @@ -27,12 +27,32 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- task: DownloadPipelineArtifact@2 - displayName: Download artifacts - inputs: - artifactName: not-signed-package - allowFailedBuilds: true - path: $(Build.SourcesDirectory)/not-signed-package +- ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion')) }}: + - download: macios + displayName: Download artifacts + artifact: not-signed-package + + # When we download an artifact from a resource pipeline, the download task places the artifact under $(Pipeline.Workspace)/$(pipeline-ref)/artifact + # in our case, it will be "$(Pipeline.Workspace)/macios/not-signed-package". Becuase this stage was originally part of the build, all the scripts + # assume that the location of the nugets is "$(Build.SourcesDirectory)/not-signed-package" already unzipped in a flat dir. This pwsh script does + # the move and flattering of the files. + - pwsh: | + $source = "$(Pipeline.Workspace)/macios/not-signed-package" + $destination = "$(Build.SourcesDirectory)/not-signed-package" + New-Item -ItemType Directory -Force -Path $destination + Write-Host "Moving content from $source to $destination" + # move all the files from the source to the destination + Get-ChildItem -Path $source -Recurse -File | Move-Item -Destination $destination + displayName: Move artifacts to the expected location + +- ${{ else }}: + + - task: DownloadPipelineArtifact@2 + displayName: Download artifacts + inputs: + artifactName: not-signed-package + allowFailedBuilds: true + path: $(Build.SourcesDirectory)/not-signed-package - pwsh: >- ./prepare_workload_apiscan.ps1 @@ -55,6 +75,7 @@ steps: preserveLogsFolder: true env: AzureServicesAuthConnectionString: RunAs=App;AppId=$(CLIENT_ID) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - task: PublishSecurityAnalysisLogs@3 displayName: Publish Security Analysis Logs