diff --git a/doc/dev/conda-builds.md b/doc/dev/conda-builds.md new file mode 100644 index 000000000000..aacebc5127ac --- /dev/null +++ b/doc/dev/conda-builds.md @@ -0,0 +1,97 @@ +# Azure SDK for Python Conda Distributions + +## Local Environment Setup + +Follow the instructions [here](https://docs.conda.io/projects/conda-build/en/latest/install-conda-build.html) to install `conda` and `conda-build`. + +## CI Build Process + +There will be a `CondaArtifact` defined in the `ci.yml` of each service directory. (`sdk/`) + +A Conda Artifact defines: +- The location of the `meta.yml` +- Which packages will be pulled into the combined artifact +- The name of the combined artifact +- Any other necessary details. + +## How to Build an Azure SDK Conda Package Locally + + +### Create Your Build Directory +Given how Conda packages are comprised of multiple source distributions _combined_, the buildable source does not exist directly within the azure-sdk-for-python repo. Currently, there is _some_ manual work that needs to be done. + +To begin, check your `ci.yml` for a `CondaArtifact`. Each these artifacts will become a single conda package. Let's use `storage/ci.yml` as an example. + +``` + - name: azure-storage + meta_source: meta.yml + common_root: azure/storage + checkout: + - package: azure-storage-blob + checkout_path: sdk/storage + version: 12.8.0 + - package: azure-storage-queue + checkout_path: sdk/storage + version: 12.1.5 + - package: azure-storage-file-share + checkout_path: sdk/storage + version: 12.4.1 + - package: azure-storage-file-datalake + checkout_path: sdk/storage + version: 12.3.0 +``` + +- `name: azure-storage`: will be the name of the "combined" sdist package that we generate. +- `meta_source: meta.yml`: this is the path (relative to the service directory) to the target conda package meta.yml. +- `common_root: azure/storage`: when generating the combined package, where will we begin combining? This is tightly bound to folder structure within the generated sdist. +- `checkout`: the `checkout` setting is a list of target packages that will go into the combined artifact. These targets will be individually sparse cloned, and copied into the conda build directory. Currently, this is a **manual step** in your local build. Reference `eng/pipelines/templates/get-tagged-code.yml` for exact details on how CI does it. + +Before we continue, you should be aware of two primary locations that are necessary, but not referenced directly in the `ci.yml`. + +The `build` folder and the `output` folder. The `build` folder (`$(Conda.Build)` variable in CI) is where we will... + +- store the cloned package code +- generate the combined sdist + +To locally repro without magic given a specific `checkout` artifact: + +``` + +git checkout `_` +grab the entire directory under the . place into your `build` folder. +``` + +Given the `storage` example. This is what your `build` folder should look like prior to invoking `build_conda_artifacts.py`. + +``` +/ + azure-storage-blob/ <-- the package directly ripped from specified tag + azure-storage-file-datalake/ + azure-storage-file-share/ + azure-storage-queue/ +``` + +### Create the Combined SDist + +Once you have a directory assembled, invoke the script to build. The below command is formatted for visibility, recombine the lines however necessary for your chosen shell environment. + + +``` +python `build_conda_artifacts.py` + -d "" + -b "" + -m "" + -r "azure/storage" + -n "azure-storage" + -s "storage" +``` + +### Generate the Conda Package + +Locally, from the anaconda prompt, set the environment variable `STORAGE_SOURCE_DISTRIBUTION` to the location of the generated sdist. After that: + +```bash +export STORAGE_SOURCE_DISTRIBUTION= +cd +conda-build . --output-folder +``` diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 76af70aa980b..f24dd9e67487 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -5,6 +5,9 @@ parameters: - name: Artifacts type: object default: [] + - name: CondaArtifacts + type: object + default: [] - name: TestPipeline type: boolean default: false @@ -61,6 +64,12 @@ jobs: TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} + - template: ../steps/build-conda-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + CondaArtifacts: ${{ parameters.CondaArtifacts }} + - job: 'Analyze' condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true')) variables: diff --git a/eng/pipelines/templates/stages/archetype-conda-release.yml b/eng/pipelines/templates/stages/archetype-conda-release.yml new file mode 100644 index 000000000000..ff9da7fda525 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-conda-release.yml @@ -0,0 +1,34 @@ +parameters: + CondaArtifacts: [] + TestPipeline: false + DependsOn: Build + ArtifactName: 'not-specified' + ServiceDirectory: 'not-specified' + + +stages: + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - ${{ each artifact in parameters.CondaArtifacts }}: + - stage: Release_${{ replace(artifact.name, '-', '_') }} + displayName: 'Conda Release: ${{artifact.name}}' + dependsOn: ${{parameters.DependsOn}} + condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) + jobs: + - deployment: CondaRelease + displayName: "Publish Conda Artifacts" + condition: ne(variables['Skip.TagRepository'], 'true') + environment: pypi + + pool: + name: azsdk-pool-mms-ubuntu-1804-general + vmImage: MMSUbuntu18.04 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Conda Artifacts diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 35fea676e8e4..5a97aa84e57b 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -5,6 +5,9 @@ parameters: - name: Artifacts type: object default: [] +- name: CondaArtifacts + type: object + default: [] - name: TestPipeline type: boolean default: false @@ -62,6 +65,7 @@ stages: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} Artifacts: ${{ parameters.Artifacts }} + CondaArtifacts: ${{ parameters.CondaArtifacts }} ${{ if eq(parameters.ServiceDirectory, 'template') }}: TestPipeline: true BeforePublishSteps: ${{ parameters.BeforePublishSteps }} @@ -85,6 +89,7 @@ stages: - ${{ each replacement in parameters.MatrixReplace }}: - ${{ replacement }} + # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - template: archetype-python-release.yml @@ -99,3 +104,15 @@ stages: TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} TargetDocRepoName: ${{ parameters.TargetDocRepoName }} DevFeedName: ${{ parameters.DevFeedName }} + + # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. + - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: + - template: archetype-conda-release.yml + parameters: + DependsOn: Build + ServiceDirectory: ${{ parameters.ServiceDirectory }} + CondaArtifacts: ${{ parameters.CondaArtifacts }} + ArtifactName: conda + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml new file mode 100644 index 000000000000..024898a4ea66 --- /dev/null +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -0,0 +1,77 @@ +parameters: + - name: TestPipeline + type: boolean + default: false + - name: ServiceDirectory + type: string + default: '' + - name: CondaArtifacts + type: object + default: [] + +steps: + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - script: | + pip install -r eng/ci_tools.txt + displayName: 'Prep Environment' + + - pwsh: | + mkdir $(Agent.BuildDirectory)/conda/ + mkdir $(Agent.BuildDirectory)/conda/output + mkdir $(Agent.BuildDirectory)/conda/build + + Write-Host "##vso[task.setvariable variable=conda.output]$(Agent.BuildDirectory)/conda/output" + Write-Host "##vso[task.setvariable variable=conda.build]$(Agent.BuildDirectory)/conda/build" + displayName: 'Create Conda Working Directories' + + - ${{ each artifact in parameters.CondaArtifacts }}: + # there may be multiple CondaArtifacts. Be certain $(conda.build) is clean just in case! + - pwsh: + Write-Host "Clean up Conda Build Directory $(conda.build)" + Remove-Item $(conda.build)/* -Recurse -Force + displayName: 'Clean Up Before Building ${{ artifact.name }}' + + - ${{ each checkout in artifact.checkout }}: + - template: /eng/pipelines/templates/steps/get-tagged-code.yml + parameters: + DestinationDirectory: $(conda.build)/${{checkout.package}} + Package: ${{checkout.package}} + CheckoutPath: ${{checkout.checkout_path}} + Version: ${{checkout.version}} + + - task: PythonScript@0 + displayName: 'Build Source Distribution for ${{ artifact.name }}' + inputs: + scriptPath: 'scripts/devops_tasks/build_conda_artifacts.py' + arguments: '-d "$(conda.output)" -b "$(conda.build)" -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" -r "${{ artifact.common_root }}" -n "${{ artifact.name }}" -s "${{ parameters.ServiceDirectory }}" -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"' + + - bash: | + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Prepend PATH with Conda and INIT' + + - bash: | + conda create --yes --quiet --name ${{ artifact.name }} + source activate ${{ artifact.name }} + conda install --yes --quiet --name ${{ artifact.name }} conda-build + displayName: 'Prepare Conda Environment for building ${{ artifact.name }}' + + - bash: | + source activate ${{ artifact.name }} + conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c https://azuresdkconda.blob.core.windows.net/channel1/ + displayName: 'Activate Conda Environment and Build ${{ artifact.name }}' + workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + + - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + parameters: + ArtifactPath: '$(Agent.BuildDirectory)/conda/output' + ArtifactName: 'conda' diff --git a/eng/pipelines/templates/steps/get-tagged-code.yml b/eng/pipelines/templates/steps/get-tagged-code.yml new file mode 100644 index 000000000000..e8182f5a5319 --- /dev/null +++ b/eng/pipelines/templates/steps/get-tagged-code.yml @@ -0,0 +1,42 @@ +parameters: + - name: DestinationDirectory + type: string + default: '' + - name: Package + type: string + default: '' + - name: CheckoutPath + type: string + default: '' + - name: Version + type: string + default: '' + +steps: + - pwsh: | + $targetPath = "$(Agent.TempDirectory)/${{ parameters.Package }}" + if (!(Test-Path $targetPath)) { + mkdir $targetPath + } + + Write-Host "##vso[task.setvariable variable=Package.Clone]$targetPath" + displayName: 'Prep for Sparse Checkout' + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "${{ parameters.CheckoutPath }}/${{ parameters.Package }}" + Repositories: + - Name: "Azure/azure-sdk-for-python" + Commitish: "${{ parameters.Package }}_${{ parameters.Version }}" + WorkingDirectory: "$(Package.Clone)" + SkipDefaultCheckout: true + + - pwsh: | + $pathForCopy = Join-Path -Path "$(Package.Clone)" -ChildPath "${{ parameters.CheckoutPath }}/${{ parameters.Package }}" + + Write-Host $pathForCopy + Write-Host ${{ parameters.DestinationDirectory }} + + Copy-Item -Path $pathForCopy -Destination ${{ parameters.DestinationDirectory }} -Recurse + displayName: 'Copy Source to Target Directory' diff --git a/scripts/devops_tasks/build_conda_artifacts.py b/scripts/devops_tasks/build_conda_artifacts.py new file mode 100644 index 000000000000..8882c8ce0edf --- /dev/null +++ b/scripts/devops_tasks/build_conda_artifacts.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# Used to generate conda artifacts given a properly formatted build folder +# +# EXAMPLE: examine the CondaArtifacts in /sdk/storage/meta.yaml +# +# Grab the source code from each of the tags in the CondaArtifact. +# +# Format the directory that you pass to the "build_directory" argument in this way +# +# /azure-storage-blob <-- package folder from tag specified +# /setup.py +# /... +# /azure-storage-queue +# /azure-storage-file-datalake +# /azure-storage-fileshare + +import argparse +import sys +import os +import shutil +import re + +from common_tasks import process_glob_string, run_check_call, str_to_bool, parse_setup +from subprocess import check_call +from distutils.dir_util import copy_tree + +VERSION_REGEX = re.compile(r"\{\%\s*set\s*version\s*=\s*\"(.*)\"\s*\%\}") + +NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: str +""" + +MANIFEST_TEMPLATE = """include *.md +{namespace_includes} +recursive-include tests *.py +recursive-include samples *.py *.md +""" + +SETUP_CFG = """ +[bdist_wheel] +universal=1 +""" + +CONDA_PKG_SETUP_TEMPLATE = """from setuptools import find_packages, setup + +setup( + name=\"{conda_package_name}\", + version=\"{version}\", + description='Microsoft Azure SDK For Python {service} Combined Conda Library', + long_description_content_type='text/markdown', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/{service}/', + classifiers=[ + "Development Status :: 5 - Production/Stable", + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'License :: OSI Approved :: MIT License', + ], + zip_safe=False, + packages=find_packages(), + install_requires=[] +) +""" + + +def create_package(pkg_directory, output_directory): + check_call( + [ + sys.executable, + "setup.py", + "sdist", + "--format", + "zip", + "-d", + output_directory, + ], + cwd=pkg_directory, + ) + + +def create_namespace_extension(target_directory): + with open(os.path.join(target_directory, "__init__.py"), "w") as f: + f.write(NAMESPACE_EXTENSION_TEMPLATE) + + +def get_pkgs_from_build_directory(build_directory, artifact_name): + return [ + os.path.join(build_directory, p) + for p in os.listdir(build_directory) + if p != artifact_name + ] + + +def create_sdist_skeleton(build_directory, artifact_name, common_root): + sdist_directory = os.path.join(build_directory, artifact_name) + + if os.path.exists(sdist_directory): + shutil.rmtree(sdist_directory) + os.makedirs(sdist_directory) + namespaces = common_root.split("/") + + # after the below function, ns_dir will be the target destination for copying from our pkgs_from_consumption + ns_dir = sdist_directory + + for ns in namespaces: + ns_dir = os.path.join(ns_dir, ns) + if not os.path.exists(ns_dir): + os.mkdir(ns_dir) + create_namespace_extension(ns_dir) + + # get all the directories in the build folder, we will pull in all of them + pkgs_for_consumption = get_pkgs_from_build_directory(build_directory, artifact_name) + + print("I see the following packages in the build directory") + print(pkgs_for_consumption) + + for pkg in pkgs_for_consumption: + pkg_till_common_root = os.path.join(pkg, common_root) + + if os.path.exists(pkg_till_common_root): + directories_for_copy = [ + file + for file in os.listdir(pkg_till_common_root) + if os.path.isdir(os.path.join(pkg_till_common_root, file)) + ] + + for directory in directories_for_copy: + src = os.path.join(pkg_till_common_root, directory) + dest = os.path.join(ns_dir, directory) + shutil.copytree(src, dest) + + +def get_version_from_meta(meta_yaml_location): + with open(os.path.abspath((meta_yaml_location)), "r") as f: + lines = f.readlines() + for line in lines: + result = VERSION_REGEX.match(line) + if result: + return result.group(1) + return "0.0.0" + + +def get_manifest_includes(common_root): + levels = common_root.split("/") + breadcrumbs = [] + breadcrumb_string = "" + + for ns in levels: + breadcrumb_string += ns + "/" + breadcrumbs.append(breadcrumb_string + "__init__.py") + + return breadcrumbs + + +def create_setup_files(build_directory, common_root, artifact_name, service, meta_yaml): + sdist_directory = os.path.join(build_directory, artifact_name) + setup_location = os.path.join(sdist_directory, "setup.py") + manifest_location = os.path.join(sdist_directory, "MANIFEST.in") + cfg_location = os.path.join(sdist_directory, "setup.cfg") + + setup_template = CONDA_PKG_SETUP_TEMPLATE.format( + conda_package_name=artifact_name, + version=get_version_from_meta(meta_yaml), + service=service, + package_excludes="'azure', 'tests', '{}'".format(common_root.replace("/", ".")), + ) + + with open(setup_location, "w") as f: + f.write(setup_template) + + manifest_template = MANIFEST_TEMPLATE.format( + namespace_includes="\n".join(["include " + ns for ns in get_manifest_includes(common_root)]) + ) + + with open(manifest_location, "w") as f: + f.write(manifest_template) + + with open(cfg_location, "w") as f: + f.write(SETUP_CFG) + + +def create_combined_sdist( + output_directory, build_directory, artifact_name, common_root, service, meta_yaml +): + singular_dependency = ( + len(get_pkgs_from_build_directory(build_directory, artifact_name)) == 0 + ) + + if not singular_dependency: + create_sdist_skeleton(build_directory, artifact_name, common_root) + create_setup_files( + build_directory, common_root, artifact_name, service, meta_yaml + ) + + sdist_location = os.path.join(build_directory, artifact_name) + + output_sdist_location = os.path.join(output_directory, "sdist", artifact_name) + + create_package(sdist_location, output_sdist_location) + output_location = os.path.join( + output_sdist_location, os.listdir(output_sdist_location)[0] + ) + + print( + "Generated Sdist for artifact {} is present at {}".format( + artifact_name, output_location + ) + ) + return output_location + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Build a Conda Package, given a properly formatted build directory, and input configuration. This script assumes that the build directory has been set up w/ the necessary sdists in each location." + ) + + parser.add_argument( + "-d", + "--distribution-directory", + dest="distribution_directory", + help="The output conda sdist will be dropped into this directory under a folder named the same as argument artifact_name.", + required=True, + ) + + parser.add_argument( + "-b", + "--build-directory", + dest="build_directory", + help="The 'working' directory. This top level path will contain all the necessary sdist code from the appropriate historical tag. EG: /azure-storage-blob, =2.18.4 + - six >=1.11.0 + run: + - python + - requests >=2.18.4 + - six >=1.11.0 + +test: + imports: + - azure.core + - azure.core.pipeline + - azure.core.pipeline.policies + - azure.core.pipeline.transport + - azure.core.polling + +about: + home: "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/core/azure-core" + license: MIT + license_family: MIT + license_file: + summary: "Microsoft Azure Core Library for Python" + doc_url: + dev_url: + +extra: + recipe-maintainers: + - lmazuel,xiangyan99 diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index ff11ea4e622d..a50fcfd3c3d5 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -60,3 +60,20 @@ extends: safeName: azuremgmtstoragesync - name: azure-mgmt-storageimportexport safeName: azuremgmtstorageimportexport + CondaArtifacts: + - name: azure-storage + meta_source: meta.yaml + common_root: azure/storage + checkout: + - package: azure-storage-blob + checkout_path: sdk/storage + version: 12.8.0 + - package: azure-storage-queue + checkout_path: sdk/storage + version: 12.1.5 + - package: azure-storage-file-share + checkout_path: sdk/storage + version: 12.4.1 + - package: azure-storage-file-datalake + checkout_path: sdk/storage + version: 12.3.0 diff --git a/sdk/storage/meta.yaml b/sdk/storage/meta.yaml new file mode 100644 index 000000000000..676f37b987a9 --- /dev/null +++ b/sdk/storage/meta.yaml @@ -0,0 +1,52 @@ +{% set name = "azure-storage" %} +{% set version = "2021.05.01" %} + +package: + name: "{{ name|lower }}" + version: "{{ version }}" + +source: + url: {{ environ.get('STORAGE_SOURCE_DISTRIBUTION', '') }} + +build: + script_env: + - STORAGE_SOURCE_DISTRIBUTION + noarch: python + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" + +requirements: + host: + - azure-core >=2021.05.01 + - cryptography >=2.1.4 + - msrest >=2021.05.01 + - pip + - python + - aiohttp + run: + - azure-core >=2021.05.01 + - cryptography >=2.1.4 + - msrest >=2021.05.01 + - python + - aiohttp + - requests-oauthlib >=0.5.0 + +test: + imports: + - azure.storage.blob + - azure.storage.queue + - azure.storage.fileshare + - azure.storage.filedatalake + +about: + home: "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/storage" + license: MIT + license_family: MIT + license_file: + summary: "Microsoft Azure Storage Client Library for Python" + doc_url: + dev_url: + +extra: + recipe-maintainers: + - your-github-id-here