diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index bc9585e9dae9..97b358e48f08 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -187,25 +187,36 @@ jobs: CondaArtifacts: ${{ parameters.CondaArtifacts}} TestProxy: ${{ parameters.TestProxy }} - - job: 'RunRegression' - condition: and(succeededOrFailed(), or(eq(variables['Run.Regression'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['System.TeamProject'],'internal')))) - displayName: 'Run Regression' - timeoutInMinutes: 180 - variables: - - template: ../variables/globals.yml - - dependsOn: - - 'Build' - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 - - steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + JobTemplatePath: /eng/pipelines/templates/jobs/regression.yml + GenerateJobName: generate_regression_matrix + SparseCheckoutPaths: [ "scripts/", "sdk/" ] + MatrixConfigs: + - Name: Python_regression_envs + Path: eng/pipelines/templates/stages/regression-job-matrix.json + Selection: sparse + GenerateVMJobs: true + PreGenerationSteps: + - script: | + pip install packaging==20.4 + displayName: 'Prep Environment' + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + - task: PythonScript@0 + displayName: 'Ensure service coverage' + inputs: + scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/update_regression_services.py' + arguments: >- + "$(TargetingString)" + --service="${{ parameters.ServiceDirectory }}" + --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/regression-job-matrix.json + CloudConfig: + Cloud: Public + DependsOn: + - 'Build' + AdditionalParameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: ../steps/test_regression.yml - parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestTimeoutInMinutes: 90 diff --git a/eng/pipelines/templates/jobs/regression.yml b/eng/pipelines/templates/jobs/regression.yml new file mode 100644 index 000000000000..6e92af32c65b --- /dev/null +++ b/eng/pipelines/templates/jobs/regression.yml @@ -0,0 +1,121 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: Matrix + type: string + - name: DependsOn + type: string + default: '' + - name: BuildStagingDirectory + type: string + default: '$(Build.ArtifactStagingDirectory)' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' + - name: UsePlatformContainer + type: boolean + default: false + - name: CloudConfig + type: object + default: {} + - name: TestTimeoutInMinutes + type: number + default: 60 + - name: BuildTargetingString + type: string + default: 'azure-*' + +# The variable $(DependentService) is set from the matrix configuration. + +jobs: + - job: + displayName: 'RegressTest' + condition: | + and( + succeededOrFailed(), + ne(${{ parameters.Matrix }}, '{}'), + or( + eq(variables['Run.Regression'], 'true'), + and( + eq(variables['System.TeamProject'], 'internal'), + eq(variables['Build.Reason'], 'Schedule') + ) + ) + ) + timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + + dependsOn: + - ${{ parameters.DependsOn }} + + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + + pool: + name: $(Pool) + vmImage: $(OSVmImage) + + ${{ if eq(parameters.UsePlatformContainer, 'true') }}: + # Add a default so the job doesn't fail when the matrix is empty + container: $[ variables['Container'] ] + + variables: + - template: ../variables/globals.yml + - name: PROXY_URL + value: "http://localhost:5000" + + # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. + steps: + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.9' + inputs: + versionSpec: '3.9' + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages' + targetPath: $(Build.ArtifactStagingDirectory) + + - script: | + pip install -r eng/regression_tools.txt + displayName: 'Prep Environment' + + - template: /eng/common/testproxy/test-proxy-tool.yml + parameters: + runProxy: false + + - template: ../steps/set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - template: ../steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + + - task: PythonScript@0 + displayName: 'Test Latest Released Dependents' + inputs: + scriptPath: 'scripts/devops_tasks/test_regression.py' + arguments: >- + "$(TargetingString)" + --service="${{ parameters.ServiceDirectory }}" + --dependent-service="$(DependentService)" + --whl-dir="${{ parameters.BuildStagingDirectory }}" + --mark-arg="not cosmosEmulator" + + - task: PythonScript@0 + displayName: 'Test Oldest Released Dependents' + inputs: + scriptPath: 'scripts/devops_tasks/test_regression.py' + arguments: >- + "$(TargetingString)" + --service="${{ parameters.ServiceDirectory }}" + --dependent-service="$(DependentService)" + --whl-dir="${{ parameters.BuildStagingDirectory }}" + --verify-latest=False + --mark-arg="not cosmosEmulator" diff --git a/eng/pipelines/templates/stages/regression-job-matrix.json b/eng/pipelines/templates/stages/regression-job-matrix.json new file mode 100644 index 000000000000..5c8726774623 --- /dev/null +++ b/eng/pipelines/templates/stages/regression-job-matrix.json @@ -0,0 +1,45 @@ +{ + "matrix": { + "Agent": { + "ubuntu-20.04": { + "OSVmImage": "MMSUbuntu20.04", + "Pool": "azsdk-pool-mms-ubuntu-2004-general" + } + }, + "DependentService": [ + "schemaregistry", + "remoterendering", + "eventhub", + "cognitivelanguage", + "tables", + "purview", + "eventgrid", + "confidentialledger", + "mixedreality", + "agrifood", + "monitor", + "attestation", + "digitaltwins", + "synapse", + "servicebus", + "keyvault", + "videoanalyzer", + "communication", + "search", + "cosmos", + "modelsrepository", + "containerregistry", + "identity", + "core", + "template", + "appconfiguration", + "webpubsub", + "textanalytics", + "metricsadvisor", + "storage", + "formrecognizer", + "translation", + "deviceupdate" + ] + } +} diff --git a/eng/pipelines/templates/steps/test_regression.yml b/eng/pipelines/templates/steps/test_regression.yml deleted file mode 100644 index bdfc5cf04ebb..000000000000 --- a/eng/pipelines/templates/steps/test_regression.yml +++ /dev/null @@ -1,55 +0,0 @@ -parameters: - ServiceDirectory: '' - BuildStagingDirectory: $(Build.ArtifactStagingDirectory) - DevFeedName: 'public/azure-sdk-for-python' - -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. -steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: '3.7' - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages' - targetPath: $(Build.ArtifactStagingDirectory) - - - script: | - pip install -r eng/regression_tools.txt - displayName: 'Prep Environment' - - - template: ../steps/set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - template: ../steps/auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} - - - task: PythonScript@0 - displayName: 'Test Latest Released Dependents' - inputs: - scriptPath: 'scripts/devops_tasks/test_regression.py' - arguments: >- - "$(TargetingString)" - --service="${{ parameters.ServiceDirectory }}" - --whl-dir="${{ parameters.BuildStagingDirectory }}" - --mark-arg="not cosmosEmulator" - - - task: PythonScript@0 - displayName: 'Test Oldest Released Dependents' - inputs: - scriptPath: 'scripts/devops_tasks/test_regression.py' - arguments: >- - "$(TargetingString)" - --service="${{ parameters.ServiceDirectory }}" - --whl-dir="${{ parameters.BuildStagingDirectory }}" - --verify-latest=False - --mark-arg="not cosmosEmulator" \ No newline at end of file diff --git a/scripts/devops_tasks/test_regression.py b/scripts/devops_tasks/test_regression.py index 59dd8d3e910d..b0d1b5efe742 100644 --- a/scripts/devops_tasks/test_regression.py +++ b/scripts/devops_tasks/test_regression.py @@ -10,6 +10,7 @@ import argparse import glob +import pdb import sys import os import logging @@ -312,22 +313,24 @@ def _is_package_installed(self, package, version): # This method identifies package dependency map for all packages in azure sdk -def find_package_dependency(glob_string, repo_root_dir): +def find_package_dependency(glob_string, repo_root_dir, dependent_service): package_paths = process_glob_string(glob_string, repo_root_dir, "", "Regression") + dependent_service_filter = os.path.join('sdk', dependent_service.lower()) + dependency_map = {} for pkg_root in package_paths: - _, _, _, requires = parse_setup(pkg_root) + if dependent_service_filter in pkg_root: + _, _, _, requires = parse_setup(pkg_root) - # Get a list of package names from install requires - required_pkgs = [parse_require(r)[0] for r in requires] - required_pkgs = [p for p in required_pkgs if p.startswith("azure")] + # Get a list of package names from install requires + required_pkgs = [parse_require(r)[0] for r in requires] + required_pkgs = [p for p in required_pkgs if p.startswith("azure")] - for req_pkg in required_pkgs: - if req_pkg not in dependency_map: - dependency_map[req_pkg] = [] - dependency_map[req_pkg].append(pkg_root) + for req_pkg in required_pkgs: + if req_pkg not in dependency_map: + dependency_map[req_pkg] = [] + dependency_map[req_pkg].append(pkg_root) - logging.info("Package dependency: {}".format(dependency_map)) return dependency_map @@ -368,7 +371,9 @@ def run_main(args): logging.info("Path {} already exists. Skipping step to clone github repo".format(code_repo_root)) # find package dependency map for azure sdk - pkg_dependency = find_package_dependency(AZURE_GLOB_STRING, code_repo_root) + pkg_dependency = find_package_dependency(AZURE_GLOB_STRING, code_repo_root, args.dependent_service) + + logging.info("Package dependency: {}".format(pkg_dependency)) # Create regression text context. One context object will be reused for all packages context = RegressionContext(args.whl_dir, temp_dir, str_to_bool(args.verify_latest), args.mark_arg) @@ -397,6 +402,13 @@ def run_main(args): "--service", help=("Name of service directory (under sdk/) to test." "Example: --service applicationinsights"), ) + + parser.add_argument( + "--dependent-service", + dest="dependent_service", + default="", + help=("Optional filter to force regression testing of only dependent packages of service X."), + ) parser.add_argument( "--whl-dir", diff --git a/scripts/devops_tasks/update_regression_services.py b/scripts/devops_tasks/update_regression_services.py new file mode 100644 index 000000000000..4c009987c7f7 --- /dev/null +++ b/scripts/devops_tasks/update_regression_services.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +import os +import argparse +import pdb +import json + +import pkg_resources +from test_regression import find_package_dependency, AZURE_GLOB_STRING +from common_tasks import process_glob_string + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + + +def parse_service(pkg_path): + path = os.path.normpath(pkg_path) + path = path.split(os.sep) + + current_segment = "" + + for path_segment in reversed(path): + if path_segment == "sdk": + return current_segment + current_segment = path_segment + + return pkg_path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Updates a given regression json file with a narrowed set of services that are dependent on the targeted service/glob string." + ) + + parser.add_argument( + "glob_string", + nargs="?", + help=( + "A comma separated list of glob strings that will target the top level directories that contain packages." + 'Examples: All = "azure*", Single = "azure-keyvault", Targeted Multiple = "azure-keyvault,azure-mgmt-resource"' + ), + ) + + parser.add_argument( + "--service", + help=("Name of service directory (under sdk/) to test." "Example: --service applicationinsights"), + ) + + parser.add_argument( + "--json", + help=("Location of the matrix configuration which has a DependentServices dimension object."), + ) + + args = parser.parse_args() + + if args.service: + service_dir = os.path.join("sdk", args.service) + target_dir = os.path.join(root_dir, service_dir) + else: + target_dir = root_dir + + targeted_packages = [ + os.path.basename(path_name) for path_name in process_glob_string(args.glob_string, target_dir, "", "Regression") + ] + deps = find_package_dependency(AZURE_GLOB_STRING, root_dir, "") + package_set = [] + + for key in list(deps.keys()): + if key not in targeted_packages: + deps.pop(key) + else: + package_set.extend(deps[key]) + + service_list = set([parse_service(pkg) for pkg in package_set]) + + try: + with open(args.json, "r") as f: + settings_json = f.read() + except FileNotFoundError as f: + print("The json file {} cannot be loaded.".format(args.json)) + exit(1) + + if len(service_list) > 0: + settings = json.loads(settings_json) + settings["matrix"]["DependentService"] = list(service_list) + json_result = json.dumps(settings) + + with open(args.json, "w") as f: + f.write(json_result) + else: + with open(args.json, "w") as f: + f.write("{}") \ No newline at end of file