Improve build times - Split X86, Parallelize Code Coverage generation (#2992)

MichelZ · web-flow · commit a6f9ce3cd5d6 · 2024-11-15T14:08:24.000-08:00
diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml
@@ -60,25 +60,51 @@ jobs:
         Get-ChildItem $(Build.SourcesDirectory)\coverageNetCore\ -Recurse -File -Filter *.coverage
       displayName: 'List coverageNetCore files [debug]'
 
-  - powershell: |
+  - pwsh: |
       dotnet tool install --global dotnet-coverage
 
-      $files = Get-ChildItem $(Build.SourcesDirectory)\coverageNetFx\ -Recurse -File -Filter *.coverage
-      # echo $files
-      mkdir coverageNetFxXml
-      $counter=0
-      foreach ($file in $files) {
-        & dotnet-coverage merge $file.FullName --output coverageNetFxXml\$((++$counter)).coveragexml --output-format xml
-      }
+      function MergeFiles {
+        param(
+        [string]$InputDirectoryPath,
+        [string]$OutputDirectoryName
+        )
+
+        $files = Get-ChildItem $InputDirectoryPath -Recurse -File -Filter *.coverage
+
+        # echo $files
+        mkdir $OutputDirectoryName
+        $counter=0
+
+        $toProcess = @()
+
+        foreach ($file in $files) {
+            $toProcess += @{ 
+                File = $file.FullName
+                OutputFile = "$OutputDirectoryName\$counter.coveragexml"
+            }
+
+            $counter++
+        }
+
+        $jobs = @()
+        foreach ($file in $toProcess){
+            $jobs += Start-ThreadJob -ScriptBlock { 
+                $params = $using:file
+                & dotnet-coverage merge $($params.File) --output $($params.OutputFile) --output-format xml 
+            } 
+        }
 
-      $files = Get-ChildItem $(Build.SourcesDirectory)\coverageNetCore\ -Recurse -File -Filter *.coverage
-      # echo $files
-      mkdir coverageNetCoreXml
-      $counter=0
-      foreach ($file in $files) {
-        & dotnet-coverage merge $file.FullName --output coverageNetCoreXml\$((++$counter)).coveragexml --output-format xml
+        Write-Host "Merging started..."
+        Wait-Job -Job $jobs
+
+        foreach ($job in $jobs) {
+            Receive-Job -Job $job -Wait -AutoRemoveJob
+        }
       }
 
+      MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetFx\" -OutputDirectoryName "coverageNetFxXml"
+      MergeFiles -InputDirectoryPath "$(Build.SourcesDirectory)\coverageNetCore\" -OutputDirectoryName "coverageNetCoreXml"
+      
       # dir coverageNetFxXml\
       # dir coverageNetCoreXml\
 
@@ -95,14 +121,28 @@ jobs:
         dir coverageNetCoreXml\
       displayName: 'List converted files [debug]'
 
-  - script: |
+  - pwsh: |
       dotnet tool install dotnet-reportgenerator-globaltool --tool-path tools
 
-      tools\reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
+      $jobs = @()
+      $jobs += Start-ThreadJob -ScriptBlock { 
+          & tools\reportgenerator "-reports:coverageNetFxXml\*.coveragexml" "-targetdir:coveragereportNetFx" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netfx\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
+      }
+
+      $jobs += Start-ThreadJob -ScriptBlock { 
+          & tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
+      }
+
+      $jobs += Start-ThreadJob -ScriptBlock { 
+          & tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
+      }
 
-      tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportAddOns" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.alwaysencrypted.azurekeyvaultprovider.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\add-ons\AzureKeyVaultProvider" "-classfilters:+Microsoft.Data.*"
+      Write-Host "Running ReportGenerator..."
+      Wait-Job -Job $jobs
 
-      tools\reportgenerator "-reports:coverageNetCoreXml\*.coveragexml" "-targetdir:coveragereportNetCore" "-reporttypes:Cobertura;" "-assemblyfilters:+microsoft.data.sqlclient.dll" "-sourcedirs:$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\netcore\src;$(Build.SourcesDirectory)\src\Microsoft.Data.SqlClient\src" "-classfilters:+Microsoft.Data.*"
+      foreach ($job in $jobs) {
+          Receive-Job -Job $job -Wait -AutoRemoveJob
+      }
     displayName: 'Run ReportGenerator'
 
   - task: PublishCodeCoverageResults@2
diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml
@@ -47,6 +47,10 @@ parameters:
     type: boolean
     default: false
 
+  - name: enableX64Test
+    type: boolean
+    default: true
+
   - name: testSet
     type: string
   
@@ -205,6 +209,7 @@ jobs:
         ${{ if ne(parameters.operatingSystem, 'Windows') }}:
           OSGroup: Unix
 
+  - ${{ if eq(parameters.enableX64Test, true) }}: # run native tests
     - template: ../steps/run-all-tests-step.yml@self # run tests
       parameters:
         debug: ${{ parameters.debug }}
diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml
@@ -65,6 +65,7 @@ stages:
                     ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}:
                       ${{ if eq(x86TF, targetFramework) }}:
                         enableX86Test: true
+                        enableX64Test: false
             - ${{ else }}: # .NET
               - ${{ each useManagedSNI in config.value.useManagedSNI }}:
                 - template: ../jobs/ci-run-tests-job.yml@self
@@ -93,6 +94,7 @@ stages:
                       ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}:
                         ${{ if eq(x86TF, targetFramework) }}:
                           enableX86Test: true
+                          enableX64Test: false
 
 - ${{ if ne(length(parameters.postTestJobs), 0) }}:
   - stage: Post_Test
diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml
@@ -107,7 +107,7 @@ stages:
 # test stages configurations
   # self hosted SQL Server on Windows
       testConfigurations:
-        windows_sql_19: # configuration name
+        windows_sql_19_x64: # configuration name
           pool: ${{parameters.defaultPoolName }} # pool name
           hostedPool: false # whether the pool is hosted or not
           images: # list of images to run tests on
@@ -137,7 +137,6 @@ stages:
             LocalDbSharedInstanceName: $(LocalDbSharedInstanceName)
             AliasName: $(SQLAliasName)
             # extra config properties
-            x86TestTargetFrameworks: [net8.0] # target frameworks should run tests on x86 as well
             SQLRootPath: $(SQL19RootPath)
             enableLocalDB: true
             # instanceName: default: MSSQLSERVER
@@ -154,7 +153,56 @@ stages:
             # localDbSharedInstanceName: default: $(LocalDbSharedInstanceName)
             # skipSqlConfiguration: # skips the SQL configuration step
 
-        windows_sql_22: 
+
+        windows_sql_19_x86: # configuration name
+          pool: ${{parameters.defaultPoolName }} # pool name
+          hostedPool: false # whether the pool is hosted or not
+          images: # list of images to run tests on
+            Win22_Sql19_x86: ADO-MMS22-SQL19 # stage display name: image name from the pool
+          TargetFrameworks: [net8.0] #[net462, net8.0] # list of target frameworks to run
+          netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }}
+          buildPlatforms: ${{parameters.buildPlatforms }}
+          testSets: ${{parameters.testSets }} # [1, 2, 3] # list of test sets to run
+          useManagedSNI: [false] # can be used for .NET Core only tests on Windows: [false, true], [false] or [true] values are allowed
+          codeCovTargetFrameworks: ${{parameters.codeCovTargetFrameworks }} # targeted frameworks that is going to participate in test result report generation
+          configSqlFor: local # setup Sql Server (local | azure | enclave)
+          operatingSystem: Windows # operating system to run tests on (Windows | Linux | Mac)
+          configProperties:
+            # config.json properties
+            TCPConnectionString: $(SQL_TCP_CONN_STRING)
+            NPConnectionString: $(SQL_NP_CONN_STRING)
+            AADAuthorityURL: $(AADAuthorityURL)
+            AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR)
+            AADServicePrincipalId: $(AADServicePrincipalId)
+            AADServicePrincipalSecret: $(AADServicePrincipalSecret)
+            AzureKeyVaultUrl: $(AzureKeyVaultUrl)
+            AzureKeyVaultTenantId: $(AzureKeyVaultTenantId)
+            SupportsIntegratedSecurity: $(SupportsIntegratedSecurity)
+            UserManagedIdentityClientId: $(UserManagedIdentityClientId)
+            FileStreamDirectory: $(FileStreamDirectory)
+            LocalDbAppName: $(LocalDbAppName)
+            LocalDbSharedInstanceName: $(LocalDbSharedInstanceName)
+            AliasName: $(SQLAliasName)
+            # extra config properties
+            x86TestTargetFrameworks: [net8.0] # target frameworks should run tests on x86
+            SQLRootPath: $(SQL19RootPath)
+            enableLocalDB: true
+            # instanceName: default: MSSQLSERVER
+            # user: default: $(user)
+            # saUser: default: $(saUser)
+            # password: default: $(password)
+            # fileStreamDirectory: default: ''
+            # x64AliasRegistryPath: default: $(x64AliasRegistryPath)
+            # x86AliasRegistryPath: default: $(x86AliasRegistryPath)
+            # SQLAliasName: default: $(SQLAliasName)
+            # SQLAliasPort: default: $(SQLAliasPort)
+            # databaseName: default: Northwind
+            # localDbAppName: default: $(LocalDbAppName)
+            # localDbSharedInstanceName: default: $(LocalDbSharedInstanceName)
+            # skipSqlConfiguration: # skips the SQL configuration step
+
+
+        windows_sql_22_x64: 
           pool: ${{parameters.defaultPoolName }}
           images:
             Win22_Sql22: ADO-MMS22-SQL22 
@@ -166,6 +214,39 @@ stages:
           codeCovTargetFrameworks: ${{parameters.codeCovTargetFrameworks }}
           configSqlFor: local
           operatingSystem: Windows
+          configProperties:
+            # config.json properties
+            TCPConnectionString: $(SQL_TCP_CONN_STRING)
+            NPConnectionString: $(SQL_NP_CONN_STRING)
+            AADAuthorityURL: $(AADAuthorityURL)
+            AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR)
+            AADServicePrincipalId: $(AADServicePrincipalId)
+            AADServicePrincipalSecret: $(AADServicePrincipalSecret)
+            AzureKeyVaultUrl: $(AzureKeyVaultUrl)
+            AzureKeyVaultTenantId: $(AzureKeyVaultTenantId)
+            SupportsIntegratedSecurity: $(SupportsIntegratedSecurity)
+            UserManagedIdentityClientId: $(UserManagedIdentityClientId)
+            FileStreamDirectory: $(FileStreamDirectory)
+            LocalDbAppName: $(LocalDbAppName)
+            LocalDbSharedInstanceName: $(LocalDbSharedInstanceName)
+            AliasName: $(SQLAliasName)
+            # extra config properties
+            SQLRootPath: $(SQL22RootPath)
+            enableLocalDB: true
+
+
+        windows_sql_22_x86: 
+          pool: ${{parameters.defaultPoolName }}
+          images:
+            Win22_Sql22_x86: ADO-MMS22-SQL22 
+          TargetFrameworks: [net462]
+          netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }}
+          buildPlatforms: ${{parameters.buildPlatforms }}
+          testSets: ${{parameters.testSets }}
+          useManagedSNI: [false]
+          codeCovTargetFrameworks: ${{parameters.codeCovTargetFrameworks }}
+          configSqlFor: local
+          operatingSystem: Windows
           configProperties:
             # config.json properties
             TCPConnectionString: $(SQL_TCP_CONN_STRING)
