Skip to content

Commit

Permalink
Disable outerloop tests on mono/mobile and fix timeout in sendtohelix…
Browse files Browse the repository at this point in the history
…help.proj (#79885)

The Helix workitem timeout still looked for the old `Outerloop` property but we're using `TestScope` instead now.
  • Loading branch information
akoeplinger authored Jul 11, 2023
1 parent 076f8c5 commit d2c5182
Show file tree
Hide file tree
Showing 30 changed files with 73 additions and 20 deletions.
14 changes: 6 additions & 8 deletions eng/pipelines/libraries/outerloop-mono.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
trigger: none

# Disabled until we get the build clean
# schedules:
# - cron: "0 8 * * *" # 8 AM UTC => 12 AM PST
# displayName: Outerloop scheduled build
# branches:
# include:
# - main
# - release/*.*
schedules:
- cron: "0 8 * * *" # 8 AM UTC => 12 AM PST
displayName: Outerloop scheduled build
branches:
include:
- main

variables:
- template: variables.yml
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,7 @@ public void AllListenersCheckCatchupList()
[InlineData(100, 102)]
[InlineData(100, 103)]
[InlineData(100, 104)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/79906", TestRuntimes.Mono)]
public void AllSubscriberStress(int numThreads, int numListenersPerThread)
{
// No listeners have been created yet
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ private static bool IsAdmin_IsNotNano_RemoteExecutorIsSupported_CanShareFiles
[ConditionalFact(nameof(IsAdmin_IsNotNano_RemoteExecutorIsSupported_CanShareFiles))] // Nano has no "netapi32.dll", Admin rights are required
[PlatformSpecific(TestPlatforms.Windows)]
[OuterLoop("Requires admin privileges")]
[ActiveIssue("https://github.com/dotnet/runtime/issues/80019", TestRuntimes.Mono)]
public void TestUserNetworkCredentialsPropertiesOnWindows()
{
const string ShareName = "testForDotNet";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,7 @@ public void TestWorkingDirectoryPropertyInChildProcess()
[ConditionalFact(nameof(IsAdmin_IsNotNano_RemoteExecutorIsSupported))] // Nano has no "netapi32.dll", Admin rights are required
[PlatformSpecific(TestPlatforms.Windows)]
[OuterLoop("Requires admin privileges")]
[ActiveIssue("https://github.com/dotnet/runtime/issues/80019", TestRuntimes.Mono)]
public void TestUserCredentialsPropertiesOnWindows()
{
using Process longRunning = CreateProcessLong();
Expand Down Expand Up @@ -1060,6 +1061,7 @@ public void StartInfo_WebPage()
[MemberData(nameof(UseShellExecute))]
[OuterLoop("Launches notepad")]
[PlatformSpecific(TestPlatforms.Windows)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)]
public void StartInfo_NotepadWithContent(bool useShellExecute)
{
string tempFile = GetTestFilePath() + ".txt";
Expand Down Expand Up @@ -1093,6 +1095,7 @@ public void StartInfo_NotepadWithContent(bool useShellExecute)
nameof(PlatformDetection.IsNotWindows8x))] // https://github.com/dotnet/runtime/issues/22007
[OuterLoop("Launches notepad")]
[PlatformSpecific(TestPlatforms.Windows)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)]
public void StartInfo_TextFile_ShellExecute()
{
// create a new extension that nobody else should be using
Expand Down Expand Up @@ -1290,6 +1293,7 @@ public void InitializeWithArgumentList_ThrowsArgumentNullException()
[MemberData(nameof(UseShellExecute))]
[OuterLoop("Launches notepad")]
[PlatformSpecific(TestPlatforms.Windows)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)]
public void StartInfo_NotepadWithContent_withArgumentList(bool useShellExecute)
{
string tempFile = GetTestFilePath() + ".txt";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public void ProcessStart_UseShellExecute_OnLinux_ThrowsIfNoProgramInstalled()
[Fact]
[OuterLoop("Opens program")]
[SkipOnPlatform(TestPlatforms.MacCatalyst, "In App Sandbox mode, the process doesn't have read access to the binary.")]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Not supported on iOS/tvOS/Android/Browser.")]
public void ProcessStart_DirectoryNameInCurDirectorySameAsFileNameInExecDirectory_Success()
{
string fileToOpen = "dotnet";
Expand Down Expand Up @@ -110,6 +111,7 @@ public void ProcessStart_DirectoryNameInCurDirectorySameAsFileNameInExecDirector

[Fact]
[OuterLoop]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Not supported on iOS/tvOS/Android/Browser.")]
public void ProcessStart_UseShellExecute_OnUnix_OpenMissingFile_DoesNotThrow()
{
if (OperatingSystem.IsLinux() &&
Expand All @@ -129,6 +131,7 @@ public void ProcessStart_UseShellExecute_OnUnix_OpenMissingFile_DoesNotThrow()

[Theory, InlineData(true), InlineData(false)]
[OuterLoop("Opens program")]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Not supported on iOS/tvOS/Android/Browser.")]
public void ProcessStart_UseShellExecute_OnUnix_SuccessWhenProgramInstalled(bool isFolder)
{
string programToOpen = s_allowedProgramsToRun.FirstOrDefault(program => IsProgramInstalled(program));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ public void TestStartWithBadWorkingDirectory()

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.HasWindowsShell))]
[OuterLoop("Launches File Explorer")]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS, "Not supported on iOS and tvOS.")]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)]
public void ProcessStart_UseShellExecute_OnWindows_OpenMissingFile_Throws()
{
string fileToOpen = Path.Combine(Environment.CurrentDirectory, "_no_such_file.TXT");
Expand All @@ -257,7 +257,7 @@ public void ProcessStart_UseShellExecute_OnWindows_OpenMissingFile_Throws()
[InlineData(true)]
[InlineData(false)]
[OuterLoop("Launches File Explorer")]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS, "Not supported on iOS and tvOS.")]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34685", TestRuntimes.Mono)]
public void ProcessStart_UseShellExecute_OnWindows_DoesNotThrow(bool isFolder)
{
string fileToOpen;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public void Ctor_ContextType()
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoNorServerCore))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData(ContextType.Machine, null)]
[InlineData(ContextType.Machine, "")]
Expand All @@ -50,6 +51,7 @@ public void Ctor_ContextType_Name(ContextType contextType, string name)
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoNorServerCore))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData(ContextType.Machine, null, null)]
[InlineData(ContextType.Machine, "", null)]
Expand All @@ -76,6 +78,7 @@ public void Ctor_ContextType_Name_Container(ContextType contextType, string name
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoNorServerCore))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData(ContextType.Machine, null, null, ContextOptions.Negotiate)]
[InlineData(ContextType.Machine, "", null, ContextOptions.Negotiate)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public void FindByTransportType_DomainNoDomainAssociatedWithoutName_ThrowsArgume
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData(DirectoryContextType.ApplicationPartition)]
[InlineData(DirectoryContextType.DirectoryServer)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public void GetDomainController_InvalidContextType_ThrowsArgumentException(Direc
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData("\0")]
[InlineData("[")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public void GetForest_NonNullNameAndNotRootedDomain_ThrowsActiveDirectoryObjectN
}

[ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34442", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop("Takes too long on domain joined machines")]
[InlineData(DirectoryContextType.Forest, "\0")]
[InlineData(DirectoryContextType.DirectoryServer, "server:port")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryData()

[Theory]
[MemberData(nameof(WriteEntry_LongFileSize_TheoryData))]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream)
{
// Write archive with a 8 Gb long entry.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryDataAsync()

[Theory]
[MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream)
{
// Write archive with a 8 Gb long entry.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public class ManagementEventWatcherTests
private const string Query = "SELECT * FROM __TimerEvent WHERE TimerID = 'MyEvent'";

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsElevatedAndSupportsWmi))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Receive_Events_Sync()
{
Expand All @@ -37,6 +38,7 @@ public void Receive_Events_Sync()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsElevatedAndSupportsWmi))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Receive_Events_Async()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public void Dynamic_Instances()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsWmiSupported))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Related_Instances()
{
Expand All @@ -46,6 +47,7 @@ public void Related_Instances()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsWmiSupported))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Relationship_Classes()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace System.Management.Tests
public class ManagementObjectSearcherTestsMofRequired
{
[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsElevatedAndSupportsWmi))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Static_Instances()
{
Expand All @@ -27,6 +28,7 @@ public void Static_Instances()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsElevatedAndSupportsWmi))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Static_Related_Instances()
{
Expand All @@ -44,6 +46,7 @@ public void Static_Related_Instances()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsElevatedAndSupportsWmi))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Static_Relationship_Classes()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public void Get_Win32_LogicalDisk()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsWmiSupported))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void GetRelated_For_Win32_LogicalDisk()
{
Expand All @@ -63,6 +64,7 @@ public void Set_Property_Win32_ComputerSystem()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsWmiSupported))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Invoke_Instance_And_Static_Method_Win32_Process()
{
Expand Down Expand Up @@ -101,6 +103,7 @@ public void Invoke_Instance_And_Static_Method_Win32_Process()
}

[ConditionalFact(typeof(WmiTestHelper), nameof(WmiTestHelper.IsWmiSupported))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34689", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[OuterLoop]
public void Serialize_ManagementException()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@ public static void GetBytes_Encoding_ReadOnlySequence_IBufferWriter_SingleSegmen

[Fact]
[OuterLoop] // this test takes ~10 seconds on modern hardware since it operates over GBs of data
[ActiveIssue("https://github.com/dotnet/runtime/issues/79883", TestRuntimes.Mono)]
public static void GetBytes_Encoding_ReadOnlySequence_IBufferWriter_LargeMultiSegment()
{
ReadOnlySequence<char> sequence = GetLargeRepeatingReadOnlySequence<char>(AllScalarsAsUtf16, 1500); // ~ 3.2bn chars of UTF-16 input
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -741,7 +741,7 @@ public async Task SendPingToExternalHostWithLowTtlTest()
options.Ttl = 1;
// This should always fail unless host is one IP hop away.
pingReply = await ping.SendPingAsync(host, TestSettings.PingTimeout, payload, options);
Assert.True(pingReply.Status == IPStatus.TimeExceeded || pingReply.Status == IPStatus.TtlExpired);
Assert.True(pingReply.Status == IPStatus.TimeExceeded || pingReply.Status == IPStatus.TtlExpired, $"pingReply.Status was {pingReply.Status} instead of TimeExceeded or TtlExpired");
Assert.NotEqual(IPAddress.Any, pingReply.Address);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ public void GetFolderPath_Unix_PersonalIsDocumentsAndUserProfile()
[InlineData(Environment.SpecialFolder.MyPictures)]
[InlineData(Environment.SpecialFolder.MyVideos)]
[InlineData(Environment.SpecialFolder.Templates)]
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Not supported on iOS/tvOS/Android/Browser.")]
public void GetFolderPath_Unix_SpecialFolderDoesNotExist_CreatesSuccessfully(Environment.SpecialFolder folder)
{
string path = Environment.GetFolderPath(folder, Environment.SpecialFolderOption.DoNotVerify);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,7 @@ public static void ToString_InvalidFormat_ThrowsFormatException()

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] // Requires a lot of memory
[OuterLoop("Takes a long time, allocates a lot of memory")]
[SkipOnMono("Frequently throws OOM on Mono")]
public static void ToString_ValidLargeFormat()
{
BigInteger b = new BigInteger(123456789000m);
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Runtime/tests/System/DoubleTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -848,6 +848,7 @@ public static void ToString_InvalidFormat_ThrowsFormatException()

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] // Requires a lot of memory
[OuterLoop("Takes a long time, allocates a lot of memory")]
[SkipOnMono("Frequently throws OOM on Mono")]
public static void ToString_ValidLargeFormat()
{
double d = 123.0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ public void TestDecryptSimple_Pkcs7Signed_GeneratedOnLinux()
VerifySimpleDecrypt(encodedMessage, certLoader, expectedContentInfo);
}

[Fact]
[ConditionalFact(typeof(PlatformSupport), nameof(PlatformSupport.IsRC2Supported))]
[OuterLoop(/* Leaks key on disk if interrupted */)]
public void TestDecryptSimpleAes256_RsaOaepSha256()
{
Expand All @@ -580,7 +580,7 @@ public void TestDecryptSimpleAes256_RsaOaepSha256()
VerifySimpleDecrypt(encodedMessage, Certificates.RSASha256KeyTransfer1, expectedContentInfo);
}

[Fact]
[ConditionalFact(typeof(PlatformSupport), nameof(PlatformSupport.IsRC2Supported))]
[OuterLoop(/* Leaks key on disk if interrupted */)]
public void TestDecryptSimpleAes256_RsaOaepSha384()
{
Expand All @@ -603,7 +603,7 @@ public void TestDecryptSimpleAes256_RsaOaepSha384()
VerifySimpleDecrypt(encodedMessage, Certificates.RSA2048Sha256KeyTransfer1, expectedContentInfo);
}

[Fact]
[ConditionalFact(typeof(PlatformSupport), nameof(PlatformSupport.IsRC2Supported))]
[OuterLoop(/* Leaks key on disk if interrupted */)]
public void TestDecryptSimpleAes256_RsaOaepSha512()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
using System.Security.Cryptography.X509Certificates;
using Xunit;

using Test.Cryptography;

namespace System.Security.Cryptography.Pkcs.EnvelopedCmsTests.Tests
{
public class DecryptTestsRsaPaddingMode : DecryptTests
Expand All @@ -16,7 +18,7 @@ public DecryptTestsRsaPaddingMode() : base(false)
{
}

[Theory]
[ConditionalTheory(typeof(PlatformSupport), nameof(PlatformSupport.IsRC2Supported))]
[MemberData(nameof(Roundtrip_RsaPaddingModes_TestData))]
[OuterLoop(/* Leaks key on disk if interrupted */)]
public static void Roundtrip_RsaPaddingModes(RSAEncryptionPadding rsaEncryptionPadding)
Expand Down
Loading

0 comments on commit d2c5182

Please sign in to comment.