System.Net.Security test stabilization

src/Common/tests/Scripts/ParallelTestExecution.ps1

@@ -0,0 +1,249 @@
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+# See the LICENSE file in the project root for more information.
+
+# Usage:
+#
+#   . ParallelTestExecution.ps1
+#   cd <testFolder> (e.g. testFolder = src\System.Net.Security\tests\FunctionalTests)
+#   RunMultiple <n> <delayVarianceMilliseconds> [-UntilFailed]
+#
+#   The above sequence will open up <n> windows running the test project present in the current folder.
+#   If -UntilFailed is used, the tests will continuously loop until a failure is detected. 
+#   Between loops, the execution will pause for <delayVarianceMilliseconds>. 
+
+function BuildAndTestBinary
+{
+    $output = (msbuild /t:rebuild,test)
+    if ($lastexitcode -ne 0)
+    {
+        throw "Build/test failed."
+    }
+
+    return $output
+}
+
+function TileWindows
+{
+    $shell = New-Object -ComObject Shell.Application
+    $shell.TileHorizontally()
+}
+
+function CurrentPath
+{
+    return (Get-Item -Path ".\" -Verbose).FullName
+}
+
+function ParseCurrentPath
+{
+    $p = CurrentPath
+
+    $test_found = $false
+    $contract_found = $false
+    $root_found = $false
+
+    while ((-not $root_found) -and ($p -ne ""))
+    {
+      $leaf = Split-Path $p -Leaf
+
+      if (Test-Path (Join-Path $p 'build.cmd'))
+      {
+          $Global:RootPath = $p
+          $root_found = $true
+      }
+
+      if ($test_found -and (-not $contract_found))
+      {
+          $Global:ContractName = $leaf
+          $contract_found = $true
+      }
+
+      if ($leaf -eq "tests")
+      {
+          $test_found = $true
+      }
+
+      $p = Split-Path $p
+    }
+
+    if (-not $test_found)
+    {
+       throw "This folder doesn't appear to be part of a test (looking for ...\contract\tests\...)." 
+    }
+}
+
+function ParseTestExecutionCommand($msBuildOutput)
+{
+    $foundTestExecution = $false
+    $cmdLine = ""
+
+    foreach ($line in $msBuildOutput)
+    {
+        if ($foundTestExecution -eq $true)
+        {
+            $cmdLine = $line
+            break
+        }
+
+        if ($line.Contains("RunTestsForProject:"))
+        {
+            $foundTestExecution = $true
+        }
+    }
+
+    if (-not $foundTestExecution)
+    {
+        throw "Cannot parse MSBuild output: please ensure that the current folder contains a test."
+    }
+
+    $Global:TestCommand = $cmdLine.Trim()
+}
+
+function ParseTestFolder($testExecutionCmdLine)
+{
+    $coreRunPath = $testExecutionCmdLine.Split()[0]
+    return Split-Path $coreRunPath
+}
+
+function Initialize
+{
+    ParseCurrentPath
+
+    Write-Host -NoNewline "Initializing tests for $($Global:ContractName) . . . "
+
+    try
+    {
+        $output = BuildAndTestBinary
+        ParseTestExecutionCommand($output)
+
+        Write-Host -ForegroundColor Green "OK"
+    }
+    catch
+    {
+        Write-Host -ForegroundColor Red "Failed"
+        throw
+    }
+}
+
+function RunOne($testCommand)
+{
+    if ($testCommand -ne "")
+    {
+        $Global:TestCommand = $testCommand
+    }
+
+    if ($Global:TestCommand -eq $null)
+    {
+        throw "Run Initialize first or pass the test command line as a parameter."
+    }
+
+    Write-Host $Global:TestCommand
+    $path = ParseTestFolder($Global:TestCommand)
+    Write-Host "$path"
+
+    Push-Location
+    cd $path
+    Invoke-Expression $Global:TestCommand
+    if ($lastexitcode -ne 0)
+    {
+        throw "Test execution failed."
+    }
+
+    Pop-Location
+}
+
+function RunUntilFailed($testCommand, $delayVarianceMilliseconds = 0)
+{
+
+    try
+    {
+        while($true)
+        {
+            RunOne $testCommand
+
+            if ($delayVarianceMilliseconds -ne 0)
+            {
+                $sleepMilliseconds = Get-Random -Minimum 0 -Maximum $delayVarianceMilliseconds
+                Write-Host -ForegroundColor Cyan "Sleeping $sleepMilliseconds"
+                Start-Sleep -Milliseconds $sleepMilliseconds
+            }
+        }
+    }
+    catch
+    {
+        Write-Host -ForegroundColor Red "Test execution failed!"
+        Read-Host "Press ENTER to continue..."
+    }
+}
+
+function RunMultiple(
+    [int]$n = 2, 
+    [int]$RandomDelayVarianceMilliseconds = 0,
+    [switch]$UntilFailed = $false)
+{  
+    if ($Global:TestCommand -eq $null)
+    {
+        Initialize
+    }
+
+    $script = $PSCommandPath
+    $testCommand = $Global:TestCommand
+
+    if ($untilFailed)
+    {
+        $executionMethod = "RunUntilFailed"
+    }
+    else
+    {
+        $executionMethod = "RunOne"
+    }
+
+    $cmdArguments = "-Command `"&{. $script; $executionMethod '$testCommand' $RandomDelayVarianceMilliseconds}`""
+
+    $processes = @()
+
+    for ($i=0; $i -lt $n; $i++)
+    {
+        $thisCmdArguments = $cmdArguments -replace ("testResults.xml", "testResults$i.xml")
+        $process = Start-Process -PassThru powershell -ArgumentList $thisCmdArguments
+        $processes += $process
+    }
+
+    $processesExited = $false
+    while (-not ([console]::KeyAvailable -or $processesExited))
+    {
+        Clear-Host
+
+        Write-Host -ForegroundColor Cyan "Active test processes:"
+        Write-Host
+        Write-Host "[Press any key to close.]"
+        Write-Host
+        $processes | Format-Table -Property Id, CPU, Handles, WS, ExitCode
+
+        $processesExited = $true
+        foreach($p in $processes)
+        {
+            if (-not $p.HasExited)
+            {
+                $processesExited = $false
+            }
+        }
+
+        Start-Sleep -Milliseconds 1000
+        TileWindows
+    }
+
+    if (-not $processesExited)
+    {
+        Write-Host -ForegroundColor Cyan "Terminating all processes."
+        foreach ($p in $processes)
+        {
+            if (-not $p.HasExited)
+            {
+                $p.Kill()
+            }
+        }
+    }
+
+    Write-Host "Done."
+}

src/Common/tests/System/Net/VirtualNetwork/VirtualNetwork.cs

@@ -10,7 +10,7 @@ namespace System.Net.Test.Common
 {
     public class VirtualNetwork
     {
-        private readonly int WaitForReadDataTimeoutMilliseconds = 10 * 1000;
+        private readonly int WaitForReadDataTimeoutMilliseconds = 30 * 1000;
 
         private readonly ConcurrentQueue<byte[]> _clientWriteQueue = new ConcurrentQueue<byte[]>();
         private readonly ConcurrentQueue<byte[]> _serverWriteQueue = new ConcurrentQueue<byte[]>();
@@ -34,9 +34,29 @@ public void ReadFrame(bool server, out byte[] buffer)
                 packetQueue = _serverWriteQueue;
             }
 
-            semaphore.Wait(WaitForReadDataTimeoutMilliseconds);
+            if (!semaphore.Wait(WaitForReadDataTimeoutMilliseconds))
+            {
+                throw new TimeoutException("VirtualNetwork: Timeout reading the next frame.");
+            }
+
+            bool dequeueSucceeded = false;
+            int remainingTries = 3;
+            int backOffDelayMilliseconds = 2;
+
+            do
+            {
+                dequeueSucceeded = packetQueue.TryDequeue(out buffer);
+                if (dequeueSucceeded)
+                {
+                    break;
+                }
+
+                remainingTries--;
+                backOffDelayMilliseconds *= backOffDelayMilliseconds;
+                Thread.Sleep(backOffDelayMilliseconds);
+            }
+            while (!dequeueSucceeded && (remainingTries > 0));
 
-            bool dequeueSucceeded = packetQueue.TryDequeue(out buffer);
             Debug.Assert(dequeueSucceeded, "Packet queue: TryDequeue failed.");
         }
 

src/Common/tests/System/Net/VirtualNetwork/VirtualNetworkStream.cs

@@ -114,16 +114,9 @@ public override Task<int> ReadAsync(byte[] buffer, int offset, int count, Cancel
 
         public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
         {
-            try
-            {
-                cancellationToken.ThrowIfCancellationRequested();
-                Write(buffer, offset, count);
-                return Task.CompletedTask;
-            }
-            catch (Exception e)
-            {
-                return Task.FromException(e);
-            }
+            return cancellationToken.IsCancellationRequested ?
+                Task.FromCanceled<int>(cancellationToken) :
+                Task.Run(() => Write(buffer, offset, count));
         }
     }
 }

src/Common/tests/Tests/System/Net/VirtualNetworkStreamTest.cs

@@ -24,10 +24,13 @@ public void VirtualNetworkStream_SingleThreadIntegrityTest_Ok()
             {
                 for (int i = 0; i < 100000; i++)
                 {
-                    int bufferSize = rnd.Next(1, 2048);
-
-                    byte[] writeFrame = new byte[bufferSize];
+                    int bufferSize;
+                    byte[] writeFrame;
+
+                    bufferSize = rnd.Next(1, 2048);
+                    writeFrame = new byte[bufferSize];
                     rnd.NextBytes(writeFrame);
+
                     uint writeChecksum = Fletcher32.Checksum(writeFrame, 0, writeFrame.Length);
                     client.Write(writeFrame, 0, writeFrame.Length);
 
@@ -99,6 +102,8 @@ public void VirtualNetworkStream_MultiThreadIntegrityTest_Ok()
             Assert.True(maxFrameSize > sizeof(int) + 1);
 
             var rnd = new Random();
+            var rndLock = new object();
+
             var network = new VirtualNetwork();
             var checksumAndLengths = new ConcurrentDictionary<int, Tuple<uint, int>>();
 
@@ -107,12 +112,21 @@ public void VirtualNetworkStream_MultiThreadIntegrityTest_Ok()
             using (var client = new VirtualNetworkStream(network, isServer: false))
             using (var server = new VirtualNetworkStream(network, isServer: true))
             {
-                Parallel.For(0, 100000, async (int i) =>
+                Parallel.For(0, 100, (int i) =>
                 {
-                    int bufferSize = rnd.Next(sizeof(int) + 1, maxFrameSize);
+                    int bufferSize;
+                    int delayMilliseconds;
+                    byte[] writeFrame;
+
+                    lock (rndLock)
+                    {
+                        bufferSize = rnd.Next(sizeof(int) + 1, maxFrameSize);
+                        delayMilliseconds = rnd.Next(0, 10);
+
+                        writeFrame = new byte[bufferSize];
+                        rnd.NextBytes(writeFrame);
+                    }
 
-                    byte[] writeFrame = new byte[bufferSize];
-                    rnd.NextBytes(writeFrame);
 
                     // First 4 bytes represent the sequence number.
                     byte[] sequenceNo = BitConverter.GetBytes(i);
@@ -123,14 +137,12 @@ public void VirtualNetworkStream_MultiThreadIntegrityTest_Ok()
 
                     checksumAndLengths.AddOrUpdate(i, writeFrameInfo, (seq, checkSum) => { Debug.Fail("Attempt to update checksum."); return new Tuple<uint, int>(0, 0); });
 
-                    await client.WriteAsync(writeFrame, 0, writeFrame.Length);
-
-                    int delayMilliseconds = rnd.Next(0, 10);
-                    await Task.Delay(delayMilliseconds);
+                    client.WriteAsync(writeFrame, 0, writeFrame.Length).GetAwaiter().GetResult();
+                    Task.Delay(delayMilliseconds).GetAwaiter().GetResult();
 
                     // First read the index to know how much data to read from this frame.
                     var readFrame = new byte[maxFrameSize];
-                    int readLen = await server.ReadAsync(readFrame, 0, maxFrameSize);
+                    int readLen = server.ReadAsync(readFrame, 0, maxFrameSize).GetAwaiter().GetResult();
 
                     int idx = BitConverter.ToInt32(readFrame, 0);
                     Tuple<uint, int> expectedFrameInfo = checksumAndLengths[idx];

src/Common/tests/project.json

@@ -18,6 +18,7 @@
     "System.Runtime.Handles": "4.0.1-rc3-24022-00",
     "System.Runtime.InteropServices": "4.1.0-rc3-24022-00",
     "System.Text.RegularExpressions": "4.1.0-rc3-24022-00",
+    "System.Threading.Thread": "4.0.0-rc3-24022-00",
     "System.Threading.Tasks": "4.0.11-rc3-24022-00",
     "System.Threading.Tasks.Parallel": "4.0.1-rc3-24022-00",
     "xunit": "2.1.0",