Updated live test to use jobs for resource cleanup (#21070)

Author: vidai-msft

.azure-pipelines/util/live-test-steps.yml

@@ -6,7 +6,7 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
-  timeoutInMinutes: 360
+  timeoutInMinutes: 600
   pool:
     vmImage: ${{ parameters.vmImage }}
 

tools/TestFx/Live/InvokeLiveTestScenarios.ps1

@@ -90,3 +90,17 @@ $liveScenarios | ForEach-Object {
     . $_.FullName
     FillLiveTestCoverageAdditionalInfo -TestCoverageDataLocation $dataLocation -BuildId $BuildId -OSVersion $OSVersion -PSVersion $PSVersion -ModuleName $moduleName
 }
+
+Write-Host "##[section]Waiting for all cleanup jobs to be completed." -ForegroundColor Green
+while (Get-Job -State Running) {
+    Write-Host "[section]Waiting for 10 seconds ..." -ForegroundColor Green
+    Start-Sleep -Seconds 10
+}
+Write-Host "##[section]All cleanup jobs are completed." -ForegroundColor Green
+
+Write-Host "##[group]Cleanup jobs information." -ForegroundColor Green
+$cleanupJobs = Get-Job
+$cleanupJobs | Select-Object Name, Command, State, PSBeginTime, PSEndTime, Output
+Write-Host "##[endgroup]"
+
+$cleanupJobs | Remove-Job

tools/TestFx/Live/LiveTestUtility.psm1

@@ -183,12 +183,18 @@ function Invoke-LiveTestCommand {
 
     do {
         try {
-            Write-Host "##[section]Start to execute the command `"$Command`"" -ForegroundColor Green
-            Write-Host "##[command]The command `"$Command`" is running" -ForegroundColor Cyan
+            Write-Host "##[section]Start to execute the command `"$Command`"." -ForegroundColor Green
+            Write-Host "##[command]The command `"$Command`" is running." -ForegroundColor Cyan
 
             $cmdResult = Invoke-Expression -Command $Command -ErrorAction Stop
 
-            Write-Host "##[section]Successfully executed the command `"$Command`"" -ForegroundColor Green
+            if ($LASTEXITCODE -eq 0) {
+                Write-Host "##[section]Successfully executed the command `"$Command`"" -ForegroundColor Green
+            }
+            else {
+                throw "Error occurred when executing the command `"$Command`"."
+            }
+
             $cmdResult
             break
         }
@@ -280,14 +286,19 @@ function Invoke-LiveTestScenario {
 
         do {
             try {
-                $sPrefs = @([psvariable]::new("ErrorActionPreference", "Stop"), [psvariable]::new("ConfirmPreference", "None"))
+                $prefs = @([psvariable]::new("ErrorActionPreference", "Stop"), [psvariable]::new("ConfirmPreference", "None"))
                 if ($snrRetryCount -eq $script:ScenarioMaxRetryCount) {
                     $prefs += [psvariable]::new("DebugPreference", "Continue")
                 }
 
                 $ScenarioScript.InvokeWithContext($null, $prefs, $snrResourceGroup)
 
-                Write-Host "##[section]Successfully executed the live scenario `"$Name`"." -ForegroundColor Green
+                if ($LASTEXITCODE -eq 0) {
+                    Write-Host "##[section]Successfully executed the live scenario `"$Name`"." -ForegroundColor Green
+                }
+                else {
+                    throw "Error occurred when executing the live scenario `"$Name`"."
+                }
                 break
             }
             catch {
@@ -377,7 +388,7 @@ function Clear-LiveTestResources {
         [string] $Name
     )
 
-    Invoke-LiveTestCommand -Command "Remove-AzResourceGroup -Name $Name -Force"
+    Invoke-LiveTestCommand -Command "Remove-AzResourceGroup -Name $Name -Force -AsJob"
 }
 
 function ConvertToLiveTestJsonErrors {