WIP | Create Extensions package

.editorconfig

@@ -14,6 +14,9 @@ indent_size = 4
 [*.{json,jsonc}]
 indent_size = 2
 
+[*.{yml,yaml}]
+indent_size = 2
+
 # C# files
 [*.cs]
 # New line preferences
@@ -160,7 +163,7 @@ indent_size = 2
 indent_size = 2
 
 # Xml files
-[*.{xml,stylecop,resx,ruleset}]
+[*.{xml,stylecop,resx,ruleset,slnx}]
 indent_size = 2
 
 # Xml config files

NuGet.config

@@ -2,10 +2,13 @@
 <configuration>
   <packageSources>
     <clear />
-    <add key="sqlclient" value="https://sqlclientdrivers.pkgs.visualstudio.com/public/_packaging/sqlclient/nuget/v3/index.json" />
+    <!-- We use a curated, vetted ADO feed for our dependencies. -->
+    <add key="curated" value="https://sqlclientdrivers.pkgs.visualstudio.com/public/_packaging/sqlclient/nuget/v3/index.json" />
   </packageSources>
   <auditSources>
     <clear />
+
+    <!-- There is no curated, vetted ADO feed for auditing, so we use the official source. -->
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
   </auditSources>
 </configuration>

build.proj

@@ -51,6 +51,7 @@
 
   <!-- Populate all managed projects -->
   <ItemGroup>
+    <ExtensionsPackage Include="src/Microsoft.Data.SqlClient.Extensions/Extensions/Extensions.slnx" />
     <SqlServerLib Include="**/Microsoft.SqlServer.Server.csproj" />
     <NetFxDriver Include="**/netfx/**/Microsoft.Data.SqlClient*.csproj" Condition="'$(IsEnabledWindows)' == 'true'" />
     <NetCoreDriver Include="**/netcore/**/Microsoft.Data.SqlClient*.csproj" />
@@ -92,6 +93,26 @@
   <Target Name="BuildTestsNetFx" DependsOnTargets="RestoreTestsNetFx;BuildAKVNetFx;BuildUnitTestsNetFx;BuildFunctionalTestsNetFx;BuildManualTestsNetFx" Condition="$(IsEnabledWindows) == 'true'"/>
   <Target Name="BuildTests" DependsOnTargets="BuildTestsNetCore;BuildTestsNetFx"/>
 
+  <!--
+    The Extensions package must be built and packed into the packages/
+    directory before the MDS projects can be restored.
+  -->
+  <Target Name="BuildExtensionsPackage">
+    <PropertyGroup>
+      <!--
+        Omit the ExtensionsPackageVersion property entirely if it is empty.
+        Otherwise, the command-line property will override the default value,
+        even if empty.
+      -->
+      <BuildProperties Condition="'$(ExtensionsPackageVersion)' != ''">ExtensionsPackageVersion=$(ExtensionsPackageVersion)</BuildProperties>
+    </PropertyGroup>
+
+    <MSBuild
+      Projects="@(ExtensionsPackage)"
+      Targets="Restore;Build;Pack"
+      Properties="$(BuildProperties)" />
+  </Target>
+
   <Target Name="RestoreSqlServerLib">
     <MSBuild Projects="@(SqlServerLib)" Targets="restore" />
   </Target>
@@ -321,7 +342,7 @@
       <TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
     </PropertyGroup>
     <Message Text=">>> Running Manual test for Windows via command: $(TestCommand)" />
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
   </Target>
 
   <!-- Run all Manual tests applicable to Unix. -->
@@ -347,13 +368,17 @@
     <Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
   </Target>
 
-  <!-- Clean -->
+  <!-- Clean all intermediate outputs. -->
   <Target Name="Clean">
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","artifacts", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","bin", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","obj", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".","packages", SearchOption.AllDirectories))' />
-    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".",".nuget", SearchOption.AllDirectories))' />
+    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "artifacts", SearchOption.AllDirectories))' />
+    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "bin", SearchOption.AllDirectories))' />
+    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "obj", SearchOption.AllDirectories))' />
+    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", ".nuget", SearchOption.AllDirectories))' />
+  </Target>
+
+  <!-- Clean all outputs, including NuGet packages. -->
+  <Target Name="CleanAll" DependsOnTargets="Clean">
+    <RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "packages", SearchOption.AllDirectories))' />
   </Target>
 
   <!-- AKV Targets ========================================================= -->

eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml

@@ -12,9 +12,13 @@ parameters:
     type: string
     default: ADO-MMS22-SQL19
 
-  - name: artifactName
+  - name: extensionsArtifactName
     type: string
-    default: Artifacts
+    default: Extensions.Artifact
+
+  - name: mdsArtifactName
+    type: string
+    default: MDS.Artifact
 
   - name: platform
     type: string
@@ -28,6 +32,11 @@ parameters:
     type: stepList
     default: []
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
 jobs:
 - job: build_nugets
 
@@ -44,12 +53,22 @@ jobs:
   - ${{ if ne(parameters.prebuildSteps, '') }}:
     - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration
 
+  # Download the Extensions package artifacts and put them in the packages/
+  # directory in the repo root.  This is where the MDS NuGet.config file will
+  # look for local packages.
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Extensions Package Artifact
+    inputs:
+      artifactName: ${{ parameters.extensionsArtifactName }}
+      targetPath: $(Build.SourcesDirectory)/packages
+
   - template: ../steps/ci-project-build-step.yml@self
     parameters:
       platform: ${{ parameters.platform }}
       configuration: ${{ parameters.configuration }}
       operatingSystem: Windows
       build: all
+      extensionsPackageVersion: ${{parameters.extensionsPackageVersion}}
 
   - template: ../steps/generate-nuget-package-step.yml@self
     parameters:
@@ -70,8 +89,8 @@ jobs:
       installNuget: false
       displayName: 'Generate NuGet package AKV Provider'
 
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: Artifacts'
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Pipeline Artifact'
     inputs:
-      PathtoPublish: $(packagePath)
-      ArtifactName: ${{ parameters.artifactName }}
+      targetPath: $(packagePath)
+      artifactName: ${{ parameters.mdsArtifactName }}

eng/pipelines/common/templates/jobs/ci-run-tests-job.yml

@@ -29,13 +29,18 @@ parameters:
     type: object
     default: {} # - key: 'value'
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
   - name: prebuildSteps
     type: stepList
     default: []
 
-  - name: artifactName
+  - name: extensionsArtifactName
     type: string
-    default: Artifacts
+    default: Extensions.Artifact
 
   - name: targetFramework
     type: string
@@ -89,6 +94,16 @@ jobs:
       value: '$(dotnetx86Path)'
 
   steps:
+
+  # Download the Extensions package artifacts and put them in the packages/
+  # directory in the repo root.  This is where the MDS NuGet.config file will
+  # look for local packages.
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Extensions Package Artifact
+    inputs:
+      artifactName: ${{ parameters.extensionsArtifactName }}
+      targetPath: $(Build.SourcesDirectory)/packages
+
   - ${{ if ne(parameters.prebuildSteps, '') }}:
     - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration
 
@@ -220,6 +235,7 @@ jobs:
         targetFramework: ${{ parameters.targetFramework }}
         referenceType: ${{ parameters.buildType }}
         testSet: ${{ parameters.testSet }}
+        extensionsPackageVersion: ${{ parameters.extensionsPackageVersion }}
         ${{ if ne(parameters.operatingSystem, 'Windows') }}:
           OSGroup: Unix
 

eng/pipelines/common/templates/stages/ci-run-tests-stage.yml

@@ -12,8 +12,8 @@ parameters:
     type: object
 
   - name: dependsOn
-    type: string
-    default: ''
+    type: object
+    default: []
 
   - name: buildType
     displayName: 'Build Type'
@@ -22,6 +22,11 @@ parameters:
       - Project
       - Package
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
   - name: prebuildSteps
     type: stepList
     default: []
@@ -34,10 +39,7 @@ stages:
 - ${{ each config in parameters.testConfigurations }}:
   - ${{ each image in config.value.images }}:
     - stage: ${{ image.key }}
-      ${{ if ne(parameters.dependsOn, '') }}:
-        dependsOn: ${{ parameters.dependsOn }}
-      ${{ else }}:
-        dependsOn: []
+      dependsOn: ${{ parameters.dependsOn }}
       jobs:
       - ${{ each targetFramework in config.value.TargetFrameworks }}:
         - ${{ each platform in config.value.buildPlatforms }}:
@@ -52,6 +54,7 @@ stages:
                   image: ${{ image.value }}
                   jobDisplayName: ${{ format('{0}_{1}_{2}', replace(targetFramework, '.', '_'), platform, testSet) }}
                   configProperties: ${{ config.value.configProperties }}
+                  extensionsPackageVersion: ${{ parameters.extensionsPackageVersion }}
                   prebuildSteps: ${{ parameters.prebuildSteps }}
                   targetFramework: ${{ targetFramework }}
                   netcoreVersionTestUtils: ${{config.value.netcoreVersionTestUtils }}
@@ -81,6 +84,7 @@ stages:
                       jobDisplayName: ${{ format('{0}_{1}_{2}_{3}', replace(targetFramework, '.', '_'), platform, 'NativeSNI', testSet) }}
                     configProperties: ${{ config.value.configProperties }}
                     useManagedSNI: ${{ useManagedSNI }}
+                    extensionsPackageVersion: ${{ parameters.extensionsPackageVersion }}
                     prebuildSteps: ${{ parameters.prebuildSteps }}                  
                     targetFramework: ${{ targetFramework }}
                     netcoreVersionTestUtils: ${{config.value.netcoreVersionTestUtils }}

eng/pipelines/common/templates/steps/build-all-tests-step.yml

@@ -32,6 +32,11 @@ parameters:
   - name: testSet
     type: string
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
 steps:
 - ${{ if contains(parameters.targetFramework, 'net4') }}: # .NET Framework
   - task: MSBuild@1
@@ -40,7 +45,7 @@ steps:
       solution: build.proj
       platform: '${{parameters.platform }}'
       configuration: '${{parameters.configuration }}'
-      msbuildArguments: '-t:BuildTestsNetFx -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
+      msbuildArguments: '-t:BuildTestsNetFx -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:ExtensionsPackageVersion=${{ parameters.extensionsPackageVersion }}.$(Build.BuildNumber)'
 
 # - ${{else if contains(parameters.targetFramework, 'netstandard')}}: # .NET Standard
 #   - task: MSBuild@1
@@ -59,7 +64,7 @@ steps:
       solution: build.proj
       platform: '${{parameters.platform }}'
       configuration: '${{parameters.configuration }}'
-      msbuildArguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }}'
+      msbuildArguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:ExtensionsPackageVersion=${{ parameters.extensionsPackageVersion }}.$(Build.BuildNumber)'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
 - ${{ else }}: # .NET on Unix
@@ -69,7 +74,7 @@ steps:
       command: custom
       projects: build.proj
       custom: msbuild
-      arguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:OSGroup=${{parameters.OSGroup }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }}'
+      arguments: '-t:BuildTestsNetCore -p:TF=${{parameters.targetFramework }} -p:TestSet=${{parameters.testSet }} -p:ReferenceType=${{parameters.referenceType }} -p:TestMicrosoftDataSqlClientVersion=${{parameters.nugetPackageVersion }} -p:OSGroup=${{parameters.OSGroup }} -p:platform=${{parameters.platform }} -p:Configuration=${{parameters.configuration }} -p:ExtensionsPackageVersion=${{ parameters.extensionsPackageVersion }}.$(Build.BuildNumber)'
       verbosityRestore: Detailed
       verbosityPack: Detailed
     condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))

eng/pipelines/common/templates/steps/ci-prebuild-step.yml

@@ -19,6 +19,11 @@ parameters:
       - Project
       - Package
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
 steps:
 - template: ensure-dotnet-version.yml
   parameters:
@@ -55,3 +60,4 @@ steps:
   - template: ci-project-build-step.yml@self
     parameters:
       build: allNoDocs
+      extensionsPackageVersion: ${{parameters.extensionsPackageVersion}}

eng/pipelines/common/templates/steps/ci-project-build-step.yml

@@ -34,6 +34,11 @@ parameters:
       - all
       - allNoDocs
 
+  - name: extensionsPackageVersion
+    displayName: Extensions Package Version Override
+    type: string
+    default: ''
+
 steps:
 - template: ./ensure-dotnet-version.yml@self
   parameters:
@@ -53,7 +58,7 @@ steps:
       inputs:
         solution: build.proj
         msbuildArchitecture: x64
-        msbuildArguments: '-t:restore'
+        msbuildArguments: '-t:restore -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
       retryCountOnTaskFailure: 1
 
     - ${{ if eq(parameters.build, 'allNoDocs') }}:
@@ -65,7 +70,7 @@ steps:
           msbuildArchitecture: x64
           platform: '${{ parameters.platform }}'
           configuration: '${{ parameters.configuration }}'
-          msbuildArguments: '-t:BuildAllConfigurations -p:GenerateDocumentationFile=false -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }}'
+          msbuildArguments: '-t:BuildAllConfigurations -p:GenerateDocumentationFile=false -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }} -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
           clean: true
 
     - ${{ if or(eq(parameters.build, 'MDS'), eq(parameters.build, 'all')) }}:
@@ -77,7 +82,7 @@ steps:
           msbuildArchitecture: x64
           platform: '${{ parameters.platform }}'
           configuration: '${{ parameters.configuration }}'
-          msbuildArguments: '-t:BuildAllConfigurations -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }}'
+          msbuildArguments: '-t:BuildAllConfigurations -p:GenerateNuGet=false -p:BuildNumber=${{ parameters.buildNumber }} -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
           clean: true
 
   - ${{ if or(eq(parameters.build, 'AKV'), eq(parameters.build, 'all'), eq(parameters.build, 'allNoDocs')) }}:
@@ -89,7 +94,7 @@ steps:
         msbuildArchitecture: x64
         platform: '${{ parameters.platform }}'
         configuration: '${{ parameters.configuration }}'
-        msbuildArguments: '-t:BuildAKVNetFx -p:BuildNumber=${{ parameters.buildNumber }}'
+        msbuildArguments: '-t:BuildAKVNetFx -p:BuildNumber=${{ parameters.buildNumber }} -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
 
     - task: MSBuild@1
       displayName: 'Build AKV Provider NetCore All OS [Win]'
@@ -99,7 +104,7 @@ steps:
         msbuildArchitecture: x64
         platform: '${{ parameters.platform }}'
         configuration: '${{ parameters.configuration }}'
-        msbuildArguments: '-t:BuildAKVNetCoreAllOS -p:BuildNumber=${{ parameters.buildNumber }}'
+        msbuildArguments: '-t:BuildAKVNetCoreAllOS -p:BuildNumber=${{ parameters.buildNumber }} -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
 
 - ${{ if or(eq(parameters.operatingSystem, 'Linux'), eq(parameters.operatingSystem, 'MacOS'), eq(parameters.operatingSystem, 'deferedToRuntime')) }}:
   - task: DotNetCoreCLI@2
@@ -109,7 +114,7 @@ steps:
       command: custom
       projects: build.proj
       custom: msbuild
-      arguments: '-t:BuildAll -p:TestEnabled=true -p:GenerateDocumentationFile=false -p:configuration=${{ parameters.configuration }}'
+      arguments: '-t:BuildAll -p:TestEnabled=true -p:GenerateDocumentationFile=false -p:configuration=${{ parameters.configuration }} -p:ExtensionsPackageVersion=${{parameters.extensionsPackageVersion}}.${{parameters.buildNumber}}'
       verbosityRestore: Detailed
       verbosityPack: Detailed
     retryCountOnTaskFailure: 1

eng/pipelines/common/templates/steps/update-nuget-config-local-feed-step.yml

@@ -34,7 +34,7 @@ steps:
     [Xml] $nugetConfig = Get-Content -Path "NuGet.config"
     $Value = Resolve-Path ${{parameters.downloadedNugetPath }}
     $newAdd = $nugetConfig.CreateElement("add")
-    $newAdd.SetAttribute("key","Package source")
+    $newAdd.SetAttribute("key","pipeline")
     $newAdd.SetAttribute("value", "$Value/" )
     $nugetConfig.configuration.packageSources.AppendChild($newAdd)
     $nugetConfig.Save("$rootFolder/NuGet.config")