[#113037209] Test speed up

AzurePowershell.Test.targets

@@ -84,7 +84,7 @@
       <XUnitTests Include=".\src\ResourceManager\ApiManagement\Commands.ApiManagement.Test\bin\Debug\Microsoft.Azure.Commands.ApiManagement.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\Profile\Commands.Profile.Test\bin\Debug\Microsoft.Azure.Commands.Profile.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\AzureBackup\Commands.AzureBackup.Test\bin\Debug\Microsoft.Azure.Commands.AzureBackup.Test.dll"/>
-      <XUnitTests Include=".\src\ResourceManager\NotificationHubs\Commands.NotificationHubs.Test\bin\Debug\Microsoft.Azure.Commands.NotificationHubs.Test.dll"/>	                      
+      <XUnitTests Include=".\src\ResourceManager\NotificationHubs\Commands.NotificationHubs.Test\bin\Debug\Microsoft.Azure.Commands.NotificationHubs.Test.dll"/>  
       <XUnitTests Include="@(AsmXUnitTests)"/>
     </ItemGroup>
     <ItemGroup Condition=" '$(scope)' == 'ServiceManagement' ">
@@ -98,19 +98,37 @@
   <Target Name="InvokeXUnit" DependsOnTargets="DeclareXunitTests">
     <Message Importance="high" Text="Running XUnit tests" />
     <MakeDir Directories="$(TestOutputDirectory)" ContinueOnError="false" />
-    <Exec 
-        Command="$(MSBuildProjectDirectory)\packages\xunit.runner.console.2.0.0\tools\xunit.console.x86.exe &quot;%(XUnitTests.Identity)&quot; -parallel none -maxthreads 0 -trait &quot;AcceptanceType=CheckIn&quot; -html &quot;$(TestOutputDirectory)\%(XUnitTests.Filename).html&quot;"
-        Timeout="$(TestTimeout)" ContinueOnError="false" Condition=" @(XUnitTests) != '' "/>
-    <OnError ExecuteTargets="TimeoutErrorHandler"/>
+
+    <xunit 
+        Assemblies="@(XUnitTests)" 
+        AppDomains="true"
+        ShadowCopy="false"
+        ParallelizeTestCollections="false"
+        ParallelizeAssemblies="true"
+        IncludeTraits="AcceptanceType=CheckIn"
+        Html="$(TestOutputDirectory)\AzurePowershellTestResults.html"
+        MaxParallelThreads="10"
+        DiagnosticMessages="false"
+        ContinueOnError="false" 
+        Condition=" @(XUnitTests) != '' "/>
   </Target>
 
   <Target Name="InvokeXUnitAll" DependsOnTargets="DeclareXunitTests">
     <Message Importance="high" Text="Running XUnit tests" />
     <MakeDir Directories="$(TestOutputDirectory)" ContinueOnError="false" />
-    <Exec 
-        Command="$(MSBuildProjectDirectory)\packages\xunit.runner.console.2.0.0\tools\xunit.console.x86.exe &quot;%(XUnitTests.Identity)&quot; -notrait &quot;RunType=LiveOnly&quot; -html &quot;$(TestOutputDirectory)\%(XUnitTests.Filename).html&quot;"
-        Timeout="$(TestTimeout)" ContinueOnError="false"/>
-    <OnError ExecuteTargets="TimeoutErrorHandler"/>
+
+    <xunit 
+        Assemblies="@(XUnitTests)" 
+        AppDomains="true"
+        ShadowCopy="false"
+        ParallelizeTestCollections="false"
+        ParallelizeAssemblies="true"
+        ExcludeTraits="RunType=LiveOnly"
+        Html="$(TestOutputDirectory)\AzurePowershellAllTestResults.html"
+        MaxParallelThreads="10"
+        DiagnosticMessages="false"
+        ContinueOnError="false" 
+        Condition=" @(XUnitTests) != '' "/>
   </Target>
 
   <Target Name="TimeoutErrorHandler">
@@ -458,7 +476,7 @@
 
   <Target Name="LiveTests"> 
     <MakeDir Directories="$(TestOutputDirectory)" ContinueOnError="false" />
-    <Exec Command="packages\xunit.runners.1.9.2\tools\xunit.console.clr4.exe  @(LiveTestDlls) /trait &quot;AcceptanceType=LiveBVT&quot; /html &quot;$(TestOutputDirectory)\Live.%(Filename).html&quot;" />
+    <Exec Command="packages\xunit.runner.console.2.1.0\tools\xunit.console.x86.exe  @(LiveTestDlls) /trait &quot;AcceptanceType=LiveBVT&quot; /html &quot;$(TestOutputDirectory)\Live.%(Filename).html&quot;" />
   </Target> 
 
 

build.proj

@@ -70,7 +70,7 @@
   <Import Condition=" $(OnPremiseBuild) " Project="$(CIToolsPath)\Microsoft.WindowsAzure.Build.OnPremise.msbuild" />
 
   <UsingTask 
-    AssemblyFile="$(MSBuildProjectDirectory)\src\packages\xunit.MSBuild.2.0.0.0\tools\xunit.runner.msbuild.dll"
+    AssemblyFile="$(MSBuildProjectDirectory)\packages\xunit.runner.msbuild.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.runner.msbuild.dll"
     TaskName="Xunit.Runner.MSBuild.xunit" />
 
   <!-- Clean the build in all configurations -->

packages.config

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="xunit.runner.console" version="2.0.0" />
-  <package id="xunit.runners" version="1.9.2" />
+  <package id="xunit.runner.console" version="2.1.0" />
+  <package id="xunit.runner.msbuild" version="2.1.0" />
 </packages>

src/Common/Commands.Common/AzureDataCmdlet.cs

@@ -95,7 +95,7 @@ protected override void PromptForDataCollectionProfileIfNotExists()
 
                 while (!this.Host.UI.RawUI.KeyAvailable && elapsedSeconds < timeToWaitInSeconds)
                 {
-                    Thread.Sleep(TimeSpan.FromMilliseconds(10));
+                    TestMockSupport.Delay(10*1000);
                     endTime = DateTime.Now;
 
                     elapsedSeconds = (endTime - startTime).TotalSeconds;

src/Common/Commands.Common/AzurePSCmdlet.cs

@@ -265,10 +265,10 @@ protected override void EndProcessing()
         protected string CurrentPath()
         {
             // SessionState is only available within PowerShell so default to
-            // the CurrentDirectory when being run from tests.
+            // the TestMockSupport.TestExecutionFolder when being run from tests.
             return (SessionState != null) ?
                 SessionState.Path.CurrentLocation.Path :
-                Environment.CurrentDirectory;
+                TestMockSupport.TestExecutionFolder;
         }
 
         protected bool IsVerbose()

src/Common/Commands.Common/TestMockSupport.cs

@@ -13,10 +13,13 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System;
 namespace Microsoft.WindowsAzure.Commands.Utilities.Common
 {
     public class TestMockSupport
     {
+        public static string TestExecutionFolder { get; set; }
+
         //a.k.a when you run under Playback mode
         public static bool RunningMocked { get; set; }
 
@@ -27,5 +30,13 @@ public static void Delay(int milliSeconds)
                 System.Threading.Thread.Sleep(milliSeconds);
             }
         }
+
+        public static void Delay(TimeSpan duration)
+        {
+            if (!RunningMocked)
+            {
+                System.Threading.Thread.Sleep(duration);
+            }
+        }
     }
 }

src/Common/Commands.ScenarioTests.Common/Commands.ScenarioTests.Common.csproj

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Import Project="..\..\packages\xunit.runner.visualstudio.2.1.0-beta4-build1109\build\net20\xunit.runner.visualstudio.props" Condition="Exists('..\..\packages\xunit.runner.visualstudio.2.1.0-beta4-build1109\build\net20\xunit.runner.visualstudio.props')" />
+  <Import Project="..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props" Condition="Exists('..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props')" />
+  <Import Project="..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props" Condition="Exists('..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props')" />
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -14,7 +15,7 @@
     <FileAlignment>512</FileAlignment>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
-    <NuGetPackageImportStamp>3c43a8cf</NuGetPackageImportStamp>
+    <NuGetPackageImportStamp>64a73b29</NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -57,10 +58,10 @@
       <HintPath>..\..\packages\Microsoft.Azure.Management.Resources.2.18.11-preview\lib\net40\Microsoft.Azure.ResourceManager.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Test.Framework">
-      <HintPath>..\..\packages\Microsoft.Azure.Test.Framework.1.0.5799.28345-prerelease\lib\net45\Microsoft.Azure.Test.Framework.dll</HintPath>
+      <HintPath>..\..\packages\Microsoft.Azure.Test.Framework.1.0.5886.28964-prerelease\lib\net45\Microsoft.Azure.Test.Framework.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Test.HttpRecorder">
-      <HintPath>..\..\packages\Microsoft.Azure.Test.HttpRecorder.1.0.5799.28345-prerelease\lib\net45\Microsoft.Azure.Test.HttpRecorder.dll</HintPath>
+      <HintPath>..\..\packages\Microsoft.Azure.Test.HttpRecorder.1.0.5886.28964-prerelease\lib\net45\Microsoft.Azure.Test.HttpRecorder.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.IdentityModel.Clients.ActiveDirectory">
       <SpecificVersion>False</SpecificVersion>
@@ -120,12 +121,20 @@
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
-    <Reference Include="xunit, Version=1.9.2.1705, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\xunit.1.9.2\lib\net20\xunit.dll</HintPath>
+    <Reference Include="xunit.abstractions, Version=2.0.0.0, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\xunit.abstractions.2.0.0\lib\net35\xunit.abstractions.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="xunit.extensions, Version=1.9.2.1705, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\xunit.extensions.1.9.2\lib\net20\xunit.extensions.dll</HintPath>
+    <Reference Include="xunit.assert, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\xunit.assert.2.1.0\lib\portable-net45+win8+wp8+wpa81\xunit.assert.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="xunit.core, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\xunit.extensibility.core.2.1.0\lib\portable-net45+win8+wp8+wpa81\xunit.core.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="xunit.execution.desktop, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\xunit.extensibility.execution.2.1.0\lib\net45\xunit.execution.desktop.dll</HintPath>
       <Private>True</Private>
     </Reference>
   </ItemGroup>

src/Common/Commands.ScenarioTests.Common/Common.ps1

@@ -280,7 +280,7 @@ function Wait-Function
 
     do
     {
-        Start-Sleep -s 5
+        Wait-Seconds 5
         $current = [DateTime]::Now
         $diff = $current - $start
         $result = &$scriptBlock
@@ -337,7 +337,7 @@ function Retry-Function
     $tries = 1;
     while(( $result -ne $true) -and ($tries -le $maxTries))
     {
-        Start-Sleep -s $interval
+        Wait-Seconds $interval
         $result = Invoke-Command -ScriptBlock $scriptBlock -ArgumentList $argument;
         $tries++;
     }

src/Common/Commands.ScenarioTests.Common/EnvironmentSetupHelper.cs

@@ -264,7 +264,7 @@ public void SetupModules(params string[] modules)
 
         public virtual Collection<PSObject> RunPowerShellTest(params string[] scripts)
         {
-            using (var powershell = System.Management.Automation.PowerShell.Create())
+            using (var powershell = System.Management.Automation.PowerShell.Create(RunspaceMode.NewRunspace))
             {
                SetupPowerShellModules(powershell);
 
@@ -302,13 +302,18 @@ public virtual Collection<PSObject> RunPowerShellTest(params string[] scripts)
 
         private void SetupPowerShellModules(System.Management.Automation.PowerShell powershell)
         {
-            powershell.AddScript(string.Format("cd \"{0}\"", Environment.CurrentDirectory));
+            powershell.AddScript("$error.clear()");
+            powershell.AddScript(string.Format("cd \"{0}\"", AppDomain.CurrentDomain.BaseDirectory));
 
             foreach (string moduleName in modules)
             {
-                powershell.AddScript(string.Format("Import-Module \".\\{0}\"", moduleName));
+                powershell.AddScript(string.Format("Import-Module \"{0}\"",
+                    Path.Combine(AppDomain.CurrentDomain.BaseDirectory, moduleName)));
             }
 
+            powershell.AddScript(
+                string.Format(@"set-location {0}", AppDomain.CurrentDomain.BaseDirectory));
+            powershell.AddScript(string.Format(@"$TestOutputRoot='{0}'", AppDomain.CurrentDomain.BaseDirectory));
             powershell.AddScript("$VerbosePreference='Continue'");
             powershell.AddScript("$DebugPreference='Continue'");
             powershell.AddScript("$ErrorActionPreference='Stop'");

src/Common/Commands.ScenarioTests.Common/PowerShellExtensions.cs

@@ -48,7 +48,7 @@ public static T GetPowerShellVariable<T>(this System.Management.Automation.Power
         }
 
         /// <summary>
-        /// Gets a powershell enumerable collection from the current session and convernts it back to it's original type.
+        /// Gets a powershell enumerable collection from the current session and converts it back to it's original type.
         /// </summary>
         /// <typeparam name="T">The powershell object original type</typeparam>
         /// <param name="powershell">The PowerShell instance</param>

src/Common/Commands.ScenarioTests.Common/Properties/AssemblyInfo.cs

@@ -34,3 +34,4 @@
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: AssemblyVersion("1.0.0.0")]
 [assembly: AssemblyFileVersion("1.0.0.0")]
+[assembly: CollectionBehavior(DisableTestParallelization = true)]

src/Common/Commands.ScenarioTests.Common/SMTestBase.cs

@@ -30,6 +30,7 @@ public abstract class SMTestBase
 
         public SMTestBase()
         {
+            System.Environment.CurrentDirectory = AppDomain.CurrentDomain.BaseDirectory;
             BaseSetup();
         }
 

src/Common/Commands.ScenarioTests.Common/packages.config

@@ -5,8 +5,8 @@
   <package id="Microsoft.Azure.Common.Authentication" version="1.6.1-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Resources" version="2.18.11-preview" targetFramework="net45" />
-  <package id="Microsoft.Azure.Test.Framework" version="1.0.5799.28345-prerelease" targetFramework="net45" />
-  <package id="Microsoft.Azure.Test.HttpRecorder" version="1.0.5799.28345-prerelease" targetFramework="net45" />
+  <package id="Microsoft.Azure.Test.Framework" version="1.0.5886.28964-prerelease" targetFramework="net45" />
+  <package id="Microsoft.Azure.Test.HttpRecorder" version="1.0.5886.28964-prerelease" targetFramework="net45" />
   <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
   <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
   <package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
@@ -16,8 +16,11 @@
   <package id="Microsoft.Rest.ClientRuntime.Azure.Authentication" version="0.11.0" targetFramework="net45" />
   <package id="Microsoft.WindowsAzure.Management" version="4.1.1" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net45" />
-  <package id="xunit" version="1.9.2" targetFramework="net45" />
-  <package id="xunit.extensions" version="1.9.2" targetFramework="net45" />
-  <package id="xunit.runner.console" version="2.0.0" targetFramework="net45" />
-  <package id="xunit.runner.visualstudio" version="2.1.0-beta4-build1109" targetFramework="net45" />
+  <package id="xunit" version="2.1.0" targetFramework="net45" />
+  <package id="xunit.abstractions" version="2.0.0" targetFramework="net45" />
+  <package id="xunit.assert" version="2.1.0" targetFramework="net45" />
+  <package id="xunit.core" version="2.1.0" targetFramework="net45" />
+  <package id="xunit.extensibility.core" version="2.1.0" targetFramework="net45" />
+  <package id="xunit.extensibility.execution" version="2.1.0" targetFramework="net45" />
+  <package id="xunit.runner.visualstudio" version="2.1.0" targetFramework="net45" />
 </packages>

src/Common/Storage/Commands.Storage.ScenarioTest/Commands.Storage.ScenarioTest.csproj

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Import Project="..\..\..\packages\xunit.runner.visualstudio.2.1.0-beta4-build1109\build\net20\xunit.runner.visualstudio.props" Condition="Exists('..\..\..\packages\xunit.runner.visualstudio.2.1.0-beta4-build1109\build\net20\xunit.runner.visualstudio.props')" />
+  <Import Project="..\..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props" Condition="Exists('..\..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props')" />
+  <Import Project="..\..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props" Condition="Exists('..\..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@@ -20,8 +21,7 @@
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <TargetFrameworkProfile />
-    <NuGetPackageImportStamp>
-    </NuGetPackageImportStamp>
+    <NuGetPackageImportStamp>75466ccb</NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -55,7 +55,7 @@
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Test.HttpRecorder">
-      <HintPath>..\..\..\packages\Microsoft.Azure.Test.HttpRecorder.1.0.5799.28345-prerelease\lib\net45\Microsoft.Azure.Test.HttpRecorder.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Test.HttpRecorder.1.0.5886.28964-prerelease\lib\net45\Microsoft.Azure.Test.HttpRecorder.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Data.Edm">
@@ -113,6 +113,22 @@
     </Reference>
     <Reference Include="System.Xml" />
     <Reference Include="System.Xml.Linq" />
+    <Reference Include="xunit.abstractions, Version=2.0.0.0, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\xunit.abstractions.2.0.0\lib\net35\xunit.abstractions.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="xunit.assert, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\xunit.assert.2.1.0\lib\portable-net45+win8+wp8+wpa81\xunit.assert.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="xunit.core, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\xunit.extensibility.core.2.1.0\lib\portable-net45+win8+wp8+wpa81\xunit.core.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="xunit.execution.desktop, Version=2.1.0.3179, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\xunit.extensibility.execution.2.1.0\lib\net45\xunit.execution.desktop.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
   </ItemGroup>
   <Choose>
     <When Condition="('$(VisualStudioVersion)' == '10.0' or '$(VisualStudioVersion)' == '') and '$(TargetFrameworkVersion)' == 'v3.5'">