User Story 38351: Add Debug CI runs

- Changed default build configuration from Release to Debug for CI-SqlClient[-Package] pipelines.
- Plumbed 'builConfiguration' template parameter down through all of the pipeline YML files.
- Removed the pipeline variable $(Configuration) in favour of buildConfiguration parameter.
- Added environment variable override for unit test config file.
- Removed triggers and schedules from existing CI pipelines.
- Added new pipeline for triggers and schedules that uses Release configuration.
- Commented out some failing assertions.
- Inhibited the ConnectionPoolTestDebug tests from running since they are flaky.
- Commented out more flaky tests.
- Doubling the test job timeout for Debug builds.
- Fixed a test that incorrectly consumes an exception when it shouldn't.
- Added diagnostics to Enclave tests that are failing to help diagnose the issues.
- Commented out a failing Debug.Assert().
- Expanded error detection in one test.
- Commented out another failing Debug.Assert().
- Fixed exception type check to look for the actual exceptions being thrown.

Author: paulmedynski

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -65,3 +65,4 @@ jobs:
     parameters:
       publishSymbols: ${{ parameters['PublishSymbols'] }}
       symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId)
+      buildConfiguration: Release

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -20,9 +20,12 @@ parameters:
     type: string
     default: $(Platform)
 
-  - name: configuration
+  - name: buildConfiguration
     type: string
-    default: $(Configuration)
+    default: Debug
+    values:
+      - Debug
+      - Release
 
   - name: prebuildSteps
     type: stepList
@@ -47,14 +50,14 @@ jobs:
   - template: ../steps/ci-project-build-step.yml@self
     parameters:
       platform: ${{ parameters.platform }}
-      configuration: ${{ parameters.configuration }}
+      buildConfiguration: ${{ parameters.buildConfiguration }}
       operatingSystem: Windows
       build: all
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
       NugetPackageVersion: $(NugetPackageVersion)
-      configuration: $(Configuration)
+      buildConfiguration: ${{ parameters.buildConfiguration }}
       nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec'
       OutputDirectory: $(packagePath)
       generateSymbolsPackage: false
@@ -63,7 +66,7 @@ jobs:
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
       NugetPackageVersion: $(NugetPackageVersion)
-      configuration: $(Configuration)
+      buildConfiguration: ${{ parameters.buildConfiguration }}
       nuspecPath: 'tools/specs/add-ons/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.nuspec'
       OutputDirectory: $(packagePath)
       generateSymbolsPackage: false

eng/pipelines/common/templates/jobs/ci-run-tests-job.yml

@@ -75,8 +75,7 @@ parameters:
 
   # The timeout, in minutes, for this job.
   - name: timeout
-    type: string
-    default: 90
+    type: number
 
 jobs:
 - job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }}

eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml

@@ -22,8 +22,7 @@ parameters:
 
   # The timeout, in minutes, for this job.
   - name: timeout
-    type: string
-    default: 90
+    type: number
 
 jobs:
 - job: run_tests_package_reference

eng/pipelines/common/templates/stages/ci-run-tests-stage.yml

@@ -31,9 +31,8 @@ parameters:
     default: []
 
   # The timeout, in minutes, for each test job.
-  - name: testsTimeout
-    type: string
-    default: 90
+  - name: testJobTimeout
+    type: number
 
 stages:
 - ${{ each config in parameters.testConfigurations }}:
@@ -52,7 +51,7 @@ stages:
                 parameters:
                   debug: ${{ parameters.debug }}
                   buildType: ${{ parameters.buildType }}
-                  timeout: ${{ parameters.testsTimeout }}
+                  timeout: ${{ parameters.testJobTimeout }}
                   poolName: ${{ config.value.pool }}
                   hostedPool: ${{ eq(config.value.hostedPool, true) }}
                   image: ${{ image.value }}
@@ -78,7 +77,7 @@ stages:
                   parameters:
                     debug: ${{ parameters.debug }}
                     buildType: ${{ parameters.buildType }}
-                    timeout: ${{ parameters.testsTimeout }}
+                    timeout: ${{ parameters.testJobTimeout }}
                     poolName: ${{ config.value.pool }}
                     hostedPool: ${{ eq(config.value.hostedPool, true) }}
                     image: ${{ image.value }}

eng/pipelines/common/templates/steps/build-all-configurations-signed-dlls-step.yml

@@ -8,9 +8,12 @@ parameters:
     type: string
     default: $(AssemblyFileVersion)
 
-  - name: Configuration
+  - name: buildConfiguration
     type: string
-    default: '$(Configuration)'
+    default: Debug
+    values:
+      - Debug
+      - Release
 
   - name: packageRefMdsVersion
     type: string
@@ -46,5 +49,5 @@ steps:
     displayName: 'BuildAllConfigurations using build.proj'
     inputs:
       solution: '**/build.proj'
-      configuration: '${{parameters.Configuration }}'
+      configuration: '${{parameters.buildConfiguration }}'
       msbuildArguments: '-p:AssemblyFileVersion=${{parameters.AssemblyFileVersion }} -t:BuildAllConfigurations -p:GenerateNuget=false -p:SigningKeyPath=$(Agent.TempDirectory)\netfxKeypair.snk'

eng/pipelines/common/templates/steps/build-all-tests-step.yml

@@ -15,9 +15,12 @@ parameters:
     type: string
     default: $(Platform)
 
-  - name: configuration
+  - name: buildConfiguration
     type: string
-    default: '$(Configuration)'
+    default: Debug
+    values:
+      - Debug
+      - Release
 
   - name: referenceType
     default: Package
@@ -39,7 +42,7 @@ steps:
     inputs:
       solution: build.proj
       platform: '${{parameters.platform }}'
-      configuration: '${{parameters.configuration }}'
+      configuration: '${{parameters.buildConfiguration }}'
       msbuildArguments: '-t:BuildTestsNetFx -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
 
 # - ${{else if contains(parameters.targetFramework, 'netstandard')}}: # .NET Standard
@@ -48,7 +51,7 @@ steps:
 #     inputs:
 #       solution: build.proj
 #       platform: '${{parameters.platform }}'
-#       configuration: '${{parameters.configuration }}'
+#       configuration: '${{parameters.buildConfiguration }}'
 #       msbuildArguments: '-t:BuildTestsNetCore -p:ReferenceType=NetStandard -p:TargetNetStandardVersion=${{parameters.targetNetStandardVersion }} -p:TF=${{parameters.targetFramework }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
 #     condition: and(succeeded(), not(startsWith(variables['TF'], 'net4')), startsWith(variables['TargetNetStandardVersion'], 'netstandard'))
 
@@ -58,7 +61,7 @@ steps:
     inputs:
       solution: build.proj
       platform: '${{parameters.platform }}'
-      configuration: '${{parameters.configuration }}'
+      configuration: '${{parameters.buildConfiguration }}'
       msbuildArguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
@@ -69,7 +72,7 @@ steps:
       command: custom
       projects: build.proj
       custom: msbuild
-      arguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:OSGroup=${{parameters.OSGroup }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }}'
+      arguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:OSGroup=${{parameters.OSGroup }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.buildConfiguration }}'
       verbosityRestore: Detailed
       verbosityPack: Detailed
     condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))

eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml

@@ -8,9 +8,12 @@ parameters:
     type: string
     default: $(TargetNetCoreVersion)
 
-  - name: configuration
+  - name: buildConfiguration
     type: string
-    default: $(Configuration)
+    default: Debug
+    values:
+      - Debug
+      - Release
 
   - name: referenceType
     default: Project
@@ -55,14 +58,14 @@ steps:
   inputs:
     solution: build.proj
     msbuildArchitecture: x64
-    msbuildArguments: '-p:Configuration=${{parameters.configuration }} -t:BuildAKVNetCore -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }}'
+    msbuildArguments: '-p:Configuration=${{parameters.buildConfiguration }} -t:BuildAKVNetCore -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }}'
 
 - task: MSBuild@1
   displayName: 'MSBuild Build Tests for ${{parameters.TargetNetCoreVersion }}'
   inputs:
     solution: build.proj
     msbuildArchitecture: x64
-    msbuildArguments: '-t:BuildTestsNetCore -p:ReferenceType=${{parameters.referenceType }} -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} -p:Configuration=${{parameters.configuration }}'
+    msbuildArguments: '-t:BuildTestsNetCore -p:ReferenceType=${{parameters.referenceType }} -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} -p:Configuration=${{parameters.buildConfiguration }}'
 
 # Don't run unit tests using package reference. Unit tests are only run using project reference.
 
@@ -71,12 +74,12 @@ steps:
   inputs:
     command: test
     projects: 'src\Microsoft.Data.SqlClient\tests\FunctionalTests\Microsoft.Data.SqlClient.FunctionalTests.csproj'
-    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }}  -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.configuration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetcoreapptests&category!=failing&category!=nonwindowstests"'
+    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }}  -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.buildConfiguration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetcoreapptests&category!=failing&category!=nonwindowstests"'
 
 - task: DotNetCoreCLI@2
   displayName: 'Run Manual Tests for ${{parameters.TargetNetCoreVersion }}'
   inputs:
     command: test
     projects: 'src\Microsoft.Data.SqlClient\tests\ManualTests\Microsoft.Data.SqlClient.ManualTesting.Tests.csproj'
-    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.configuration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter  category!=nonnetcoreapptests&category!=failing&category!=nonwindowstests  --collect "Code Coverage"'
+    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetCoreVersion=${{parameters.TargetNetCoreVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.buildConfiguration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter  category!=nonnetcoreapptests&category!=failing&category!=nonwindowstests  --collect "Code Coverage"'
   retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}

eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml

@@ -8,9 +8,12 @@ parameters:
     type: string
     default: $(TargetNetFxVersion)
 
-  - name: configuration
+  - name: buildConfiguration
     type: string
-    default: $(Configuration)
+    default: Debug
+    values:
+      - Debug
+      - Release
 
   - name: referenceType
     default: Project
@@ -55,13 +58,13 @@ steps:
   inputs:
     solution: build.proj
     msbuildArchitecture: x64
-    msbuildArguments: '-p:Configuration=${{parameters.configuration }} -t:BuildAKVNetFx -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }}'
+    msbuildArguments: '-p:Configuration=${{parameters.buildConfiguration }} -t:BuildAKVNetFx -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }}'
 
 - task: MSBuild@1
   displayName: 'MSBuild Build Tests for ${{parameters.TargetNetFxVersion }}'
   inputs:
     solution: build.proj
-    msbuildArguments: ' -t:BuildTestsNetFx -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:Configuration=${{parameters.configuration }} -p:Platform=${{parameters.platform }}'
+    msbuildArguments: ' -t:BuildTestsNetFx -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:Configuration=${{parameters.buildConfiguration }} -p:Platform=${{parameters.platform }}'
 
 # Don't run unit tests using package reference. Unit tests are only run using project reference.
 
@@ -70,12 +73,12 @@ steps:
   inputs:
     command: test
     projects: 'src\Microsoft.Data.SqlClient\tests\FunctionalTests\Microsoft.Data.SqlClient.FunctionalTests.csproj'
-    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.configuration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetfxtests&category!=failing&category!=nonwindowstests" --collect "Code Coverage"'
+    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.buildConfiguration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetfxtests&category!=failing&category!=nonwindowstests" --collect "Code Coverage"'
 
 - task: DotNetCoreCLI@2
   displayName: 'Run Manual Tests for ${{parameters.TargetNetFxVersion }}'
   inputs:
     command: test
     projects: 'src\Microsoft.Data.SqlClient\tests\ManualTests\Microsoft.Data.SqlClient.ManualTesting.Tests.csproj'
-    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.configuration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetfxtests&category!=failing&category!=nonwindowstests" --collect "Code Coverage"'
+    arguments: '-p:Platform=${{parameters.platform }} -p:TestTargetOS="${{parameters.TestTargetOS }}" -p:TargetNetFxVersion=${{parameters.TargetNetFxVersion }} -p:ReferenceType=${{parameters.referenceType }} -p:Configuration=${{parameters.buildConfiguration }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.NugetPackageVersion }} --no-build -v n --filter "category!=nonnetfxtests&category!=failing&category!=nonwindowstests" --collect "Code Coverage"'
   retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }}

eng/pipelines/common/templates/steps/ci-project-build-step.yml

@@ -8,10 +8,13 @@ parameters:
     type: string
     default: $(Platform)
 
-  - name: configuration
+  - name: buildConfiguration
     type: string
-    default: $(Configuration)
-  
+    default: Debug
+    values:
+      - Debug
+      - Release
+
   - name: buildNumber
     type: string
     default: $(BuildNumber)
@@ -64,7 +67,7 @@ steps:
           solution: build.proj
           msbuildArchitecture: x64
           platform: '${{ parameters.platform }}'
-          configuration: '${{ parameters.configuration }}'
+          configuration: '${{ parameters.buildConfiguration }}'
           msbuildArguments: '-t:BuildAllConfigurations -p:GenerateDocumentationFile=false -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }}'
           clean: true
 
@@ -76,7 +79,7 @@ steps:
           solution: build.proj
           msbuildArchitecture: x64
           platform: '${{ parameters.platform }}'
-          configuration: '${{ parameters.configuration }}'
+          configuration: '${{ parameters.buildConfiguration }}'
           msbuildArguments: '-t:BuildAllConfigurations -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }}'
           clean: true
 
@@ -88,7 +91,7 @@ steps:
         solution: build.proj
         msbuildArchitecture: x64
         platform: '${{ parameters.platform }}'
-        configuration: '${{ parameters.configuration }}'
+        configuration: '${{ parameters.buildConfiguration }}'
         msbuildArguments: '-t:BuildAKVNetFx -p:BuildNumber=${{ parameters.buildNumber }}'
 
     - task: MSBuild@1
@@ -98,7 +101,7 @@ steps:
         solution: build.proj
         msbuildArchitecture: x64
         platform: '${{ parameters.platform }}'
-        configuration: '${{ parameters.configuration }}'
+        configuration: '${{ parameters.buildConfiguration }}'
         msbuildArguments: '-t:BuildAKVNetCoreAllOS -p:BuildNumber=${{ parameters.buildNumber }}'
 
 - ${{ if or(eq(parameters.operatingSystem, 'Linux'), eq(parameters.operatingSystem, 'MacOS'), eq(parameters.operatingSystem, 'deferedToRuntime')) }}:
@@ -109,7 +112,7 @@ steps:
       command: custom
       projects: build.proj
       custom: msbuild
-      arguments: '-t:BuildAll -p:TestEnabled=true -p:GenerateDocumentationFile=false -p:configuration=${{ parameters.configuration }}'
+      arguments: '-t:BuildAll -p:TestEnabled=true -p:GenerateDocumentationFile=false -p:configuration=${{ parameters.buildConfiguration }}'
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 1