Add Conda Publishing (#17889)

* Add Conda Build Steps and supporting infrastructure. 
* Add meta.yml and extensions to ci.yml in both sdk/core and sdk/storage
* Add placeholder Conda release stage

Co-authored-by: Sean Kane <68240067+seankane-msft@users.noreply.github.com>

doc/dev/conda-builds.md

@@ -0,0 +1,97 @@
+# Azure SDK for Python Conda Distributions
+
+## Local Environment Setup
+
+Follow the instructions [here](https://docs.conda.io/projects/conda-build/en/latest/install-conda-build.html) to install `conda` and `conda-build`.
+
+## CI Build Process
+
+There will be a `CondaArtifact` defined in the `ci.yml` of each service directory. (`sdk/<service>`)
+
+A Conda Artifact defines:
+- The location of the `meta.yml`
+- Which packages will be pulled into the combined artifact
+- The name of the combined artifact
+- Any other necessary details.
+
+## How to Build an Azure SDK Conda Package Locally
+
+
+### Create Your Build Directory
+Given how Conda packages are comprised of multiple source distributions _combined_, the buildable source does not exist directly within the azure-sdk-for-python repo. Currently, there is _some_ manual work that needs to be done.
+
+To begin, check your `ci.yml` for a `CondaArtifact`. Each these artifacts will become a single conda package. Let's use `storage/ci.yml` as an example.
+
+```
+    - name: azure-storage
+      meta_source: meta.yml
+      common_root: azure/storage
+      checkout:
+        - package: azure-storage-blob
+          checkout_path: sdk/storage
+          version: 12.8.0
+        - package: azure-storage-queue
+          checkout_path: sdk/storage
+          version: 12.1.5
+        - package: azure-storage-file-share
+          checkout_path: sdk/storage
+          version: 12.4.1
+        - package: azure-storage-file-datalake
+          checkout_path: sdk/storage
+          version: 12.3.0
+```
+
+- `name: azure-storage`: will be the name of the "combined" sdist package that we generate.
+- `meta_source: meta.yml`: this is the path (relative to the service directory) to the target conda package meta.yml.
+- `common_root: azure/storage`: when generating the combined package, where will we begin combining? This is tightly bound to folder structure within the generated sdist.
+- `checkout`: the `checkout` setting is a list of target packages that will go into the combined artifact. These targets will be individually sparse cloned, and copied into the conda build directory. Currently, this is a **manual step** in your local build. Reference `eng/pipelines/templates/get-tagged-code.yml` for exact details on how CI does it.
+
+Before we continue, you should be aware of two primary locations that are necessary, but not referenced directly in the `ci.yml`.
+
+The `build` folder and the `output` folder. The `build` folder (`$(Conda.Build)` variable in CI) is where we will...
+
+- store the cloned package code
+- generate the combined sdist
+
+To locally repro without magic given a specific `checkout` artifact:
+
+```
+<cd sdk-for-python>
+git checkout `<package>_<version>`
+grab the entire <package> directory under the <checkout_path>. place into your `build` folder.
+```
+
+Given the `storage` example. This is what your `build` folder should look like prior to invoking `build_conda_artifacts.py`.
+
+```
+<build directory>/
+    azure-storage-blob/          <-- the package directly ripped from specified tag
+    azure-storage-file-datalake/
+    azure-storage-file-share/
+    azure-storage-queue/
+```
+
+### Create the Combined SDist
+
+Once you have a directory assembled, invoke the script to build. The below command is formatted for visibility, recombine the lines however necessary for your chosen shell environment.
+
+
+```
+python `build_conda_artifacts.py`
+    -d "<output folder>"
+    -b "<build folder>"
+    -m "<resolvable path to sdk/storage/meta.yml>"
+    -r "azure/storage"
+    -n "azure-storage"
+    -s "storage"
+```
+
+### Generate the Conda Package
+
+Locally, from the anaconda prompt, set the environment variable `STORAGE_SOURCE_DISTRIBUTION` to the location of the generated sdist. After that:
+
+```bash
+export STORAGE_SOURCE_DISTRIBUTION=<path/to/generated/sdist>
+cd <meta.yml directory>
+conda-build . --output-folder <conda.output>
+```

eng/pipelines/templates/jobs/ci.yml

@@ -5,6 +5,9 @@ parameters:
   - name: Artifacts
     type: object
     default: []
+  - name: CondaArtifacts
+    type: object
+    default: []
   - name: TestPipeline
     type: boolean
     default: false
@@ -61,6 +64,12 @@ jobs:
         TestPipeline: ${{ parameters.TestPipeline }}
         Artifacts: ${{ parameters.Artifacts }}
 
+    - template: ../steps/build-conda-artifacts.yml
+      parameters:
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        TestPipeline: ${{ parameters.TestPipeline }}
+        CondaArtifacts: ${{ parameters.CondaArtifacts }}
+
   - job: 'Analyze'
     condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true'))
     variables:

eng/pipelines/templates/stages/archetype-conda-release.yml

@@ -0,0 +1,34 @@
+parameters:
+  CondaArtifacts: []
+  TestPipeline: false
+  DependsOn: Build
+  ArtifactName: 'not-specified'
+  ServiceDirectory: 'not-specified'
+
+
+stages:
+  - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
+    - ${{ each artifact in parameters.CondaArtifacts }}:
+      - stage: Release_${{ replace(artifact.name, '-', '_') }}
+        displayName: 'Conda Release: ${{artifact.name}}'
+        dependsOn: ${{parameters.DependsOn}}
+        condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'))
+        jobs:
+          - deployment: CondaRelease
+            displayName: "Publish Conda Artifacts"
+            condition: ne(variables['Skip.TagRepository'], 'true')
+            environment: pypi
+
+            pool:
+              name: azsdk-pool-mms-ubuntu-1804-general
+              vmImage: MMSUbuntu18.04
+
+            strategy:
+              runOnce:
+                deploy:
+                  steps:
+                    - checkout: self
+                    - pwsh: |
+                        Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}
+                      workingDirectory: $(Pipeline.Workspace)
+                      displayName: Output Visible Conda Artifacts

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -5,6 +5,9 @@ parameters:
 - name: Artifacts
   type: object
   default: []
+- name: CondaArtifacts
+  type: object
+  default: []
 - name: TestPipeline
   type: boolean
   default: false
@@ -62,6 +65,7 @@ stages:
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         Artifacts: ${{ parameters.Artifacts }}
+        CondaArtifacts: ${{ parameters.CondaArtifacts }}
         ${{ if eq(parameters.ServiceDirectory, 'template') }}:
           TestPipeline: true
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
@@ -85,6 +89,7 @@ stages:
           - ${{ each replacement in parameters.MatrixReplace }}:
             - ${{ replacement }}
 
+
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
     - template: archetype-python-release.yml
@@ -99,3 +104,15 @@ stages:
         TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }}
         TargetDocRepoName: ${{ parameters.TargetDocRepoName }}
         DevFeedName: ${{ parameters.DevFeedName }}
+
+  # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
+  - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
+    - template: archetype-conda-release.yml
+      parameters:
+        DependsOn: Build
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        CondaArtifacts: ${{ parameters.CondaArtifacts }}
+        ArtifactName: conda
+        ${{ if eq(parameters.ServiceDirectory, 'template') }}:
+          TestPipeline: true
+

eng/pipelines/templates/steps/build-conda-artifacts.yml

@@ -0,0 +1,77 @@
+parameters:
+  - name: TestPipeline
+    type: boolean
+    default: false
+  - name: ServiceDirectory
+    type: string
+    default: ''
+  - name: CondaArtifacts
+    type: object
+    default: []
+
+steps:
+  - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
+    parameters:
+      PackageName: "azure-template"
+      ServiceDirectory: "template"
+      TestPipeline: ${{ parameters.TestPipeline }}
+
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(PythonVersion)'
+    inputs:
+      versionSpec: $(PythonVersion)
+
+  - script: |
+      pip install -r eng/ci_tools.txt
+    displayName: 'Prep Environment'
+
+  - pwsh: |
+      mkdir $(Agent.BuildDirectory)/conda/
+      mkdir $(Agent.BuildDirectory)/conda/output
+      mkdir $(Agent.BuildDirectory)/conda/build
+
+      Write-Host "##vso[task.setvariable variable=conda.output]$(Agent.BuildDirectory)/conda/output"
+      Write-Host "##vso[task.setvariable variable=conda.build]$(Agent.BuildDirectory)/conda/build"
+    displayName: 'Create Conda Working Directories'
+
+  - ${{ each artifact in parameters.CondaArtifacts }}:
+    # there may be multiple CondaArtifacts. Be certain $(conda.build) is clean just in case!
+    - pwsh:
+        Write-Host "Clean up Conda Build Directory $(conda.build)"
+        Remove-Item $(conda.build)/* -Recurse -Force
+      displayName: 'Clean Up Before Building ${{ artifact.name }}'
+
+    - ${{ each checkout in artifact.checkout }}:
+      - template: /eng/pipelines/templates/steps/get-tagged-code.yml
+        parameters:
+          DestinationDirectory: $(conda.build)/${{checkout.package}}
+          Package: ${{checkout.package}}
+          CheckoutPath: ${{checkout.checkout_path}}
+          Version: ${{checkout.version}}
+
+    - task: PythonScript@0
+      displayName: 'Build Source Distribution for ${{ artifact.name }}'
+      inputs:
+        scriptPath: 'scripts/devops_tasks/build_conda_artifacts.py'
+        arguments: '-d "$(conda.output)" -b "$(conda.build)" -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" -r "${{ artifact.common_root }}" -n "${{ artifact.name }}" -s "${{ parameters.ServiceDirectory }}" -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"'
+
+    - bash: |
+        echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: 'Prepend PATH with Conda and INIT'
+
+    - bash: |
+        conda create --yes --quiet --name ${{ artifact.name }}
+        source activate ${{ artifact.name }}
+        conda install --yes --quiet --name ${{ artifact.name }} conda-build
+      displayName: 'Prepare Conda Environment for building ${{ artifact.name }}'
+
+    - bash: |
+        source activate ${{ artifact.name }}
+        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c https://azuresdkconda.blob.core.windows.net/channel1/
+      displayName: 'Activate Conda Environment and Build ${{ artifact.name }}'
+      workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+  
+  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+    parameters:
+      ArtifactPath: '$(Agent.BuildDirectory)/conda/output'
+      ArtifactName: 'conda'

eng/pipelines/templates/steps/get-tagged-code.yml

@@ -0,0 +1,42 @@
+parameters:
+  - name: DestinationDirectory
+    type: string
+    default: ''
+  - name: Package
+    type: string
+    default: ''
+  - name: CheckoutPath
+    type: string
+    default: ''
+  - name: Version
+    type: string
+    default: ''
+
+steps:
+  - pwsh: |
+      $targetPath = "$(Agent.TempDirectory)/${{ parameters.Package }}"
+      if (!(Test-Path $targetPath)) {
+        mkdir $targetPath
+      }
+
+      Write-Host "##vso[task.setvariable variable=Package.Clone]$targetPath"
+    displayName: 'Prep for Sparse Checkout'
+
+  - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+    parameters:
+      Paths:
+        - "${{ parameters.CheckoutPath }}/${{ parameters.Package }}"
+      Repositories:
+        - Name: "Azure/azure-sdk-for-python"
+          Commitish: "${{ parameters.Package }}_${{ parameters.Version }}"
+          WorkingDirectory: "$(Package.Clone)"
+      SkipDefaultCheckout: true
+
+  - pwsh: |
+      $pathForCopy = Join-Path -Path "$(Package.Clone)" -ChildPath "${{ parameters.CheckoutPath }}/${{ parameters.Package }}"
+      
+      Write-Host $pathForCopy
+      Write-Host ${{ parameters.DestinationDirectory }}
+
+      Copy-Item -Path $pathForCopy -Destination ${{ parameters.DestinationDirectory }} -Recurse
+    displayName: 'Copy Source to Target Directory'