Merge from upstream master and adjust code accordingly to incorporate with OOB changes. (param renames e.g. _lock_expired)

Author: KieranBrantnerMagee

doc/dev/mgmt/tests.md

@@ -176,7 +176,7 @@ providing several features useful for the SDK tests, for example:
 *   Patches for overriding functions and methods that don't work well with tests
     (such as long-running operations)
 
-Code in the [`azure-sdk-tools/devtools_testutils`](tools/azure-sdk-tools/devtools_testutils) directory
+Code in the [`azure-sdk-tools/devtools_testutils`](https://github.com/Azure/azure-sdk-for-python/tree/master/tools/azure-sdk-tools/devtools_testutils) directory
 provides concrete implementations of the features provided in `scenario_tests`
 that are oriented around use in SDK testing
 and that you can use directly in your unit tests.

eng/ci_tools.txt

@@ -1,36 +1,35 @@
 # requirements leveraged by ci tools
 setuptools==44.1.0; python_version == '2.7'
-setuptools==45.1.0; python_version >= '3.5'
+setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
 wheel==0.34.2 
-Jinja2==2.11.1
+Jinja2==2.11.2
 packaging==20.4
-tox==3.14.6
+tox==3.15.0
 tox-monorepo==0.1.2
-twine==1.15.0
+twine==1.15.0; python_version == '2.7' or python_version == '3.5'
+twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
 readme-renderer[md]==25.0
-doc-warden==0.5.4
+doc-warden==0.7.1
+# we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
-codecov==2.0.22 
-beautifulsoup4==4.8.2
+codecov==2.1.0
+beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 pyOpenSSL==19.1.0
 json-delta==2.0
-ConfigArgParse==1.1
+ConfigArgParse==1.2.3
 six==1.14.0
 vcrpy==3.0.0
 pyyaml==5.3.1
-pytest==5.4.1; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
 pytest-cov==2.8.1
 
 # local dev packages
 ./tools/azure-devtools
 ./tools/azure-sdk-tools
-
-
-

eng/pipelines/templates/steps/analyze.yml

@@ -91,8 +91,8 @@ steps:
       BuildTargetingString: ${{ parameters.BuildTargetingString }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
 
-  - template: ../steps/run_apistub.yml
-    parameters: 
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
-      BuildTargetingString: ${{ parameters.BuildTargetingString }}
-      TestMarkArgument: ${{ parameters.TestMarkArgument }}
+  # - template: ../steps/run_apistub.yml
+  #   parameters:
+  #     ServiceDirectory: ${{ parameters.ServiceDirectory }}
+  #     BuildTargetingString: ${{ parameters.BuildTargetingString }}
+  #     TestMarkArgument: ${{ parameters.TestMarkArgument }}

eng/test_tools.txt

@@ -1,29 +1,29 @@
 # requirements leveraged by ci for testing
-pytest==5.4.1; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
-pytest-asyncio==0.10.0; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
+pytest-asyncio==0.12.0; python_version >= '3.5'
 pytest-cov==2.8.1
 pytest-custom-exit-code==0.3.0
-pytest-xdist==1.31.0
+pytest-xdist==1.32.0
+# we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 readme-renderer[md]==25.0
 pyOpenSSL==19.1.0
 json-delta==2.0
-ConfigArgParse==1.1
+ConfigArgParse==1.2.3
 six==1.14.0
 vcrpy==3.0.0
 pyyaml==5.3.1
 packaging==20.4
 wheel==0.34.2
-Jinja2==2.11.1
+Jinja2==2.11.2
 
 # Locking pylint and required packages
-pylint==2.3.1; python_version >= '3.4'
 pylint==1.8.4; python_version < '3.4'
-astroid==2.3.3; python_version >= '3.4'
+pylint==2.5.2; python_version >= '3.4'
 
 ../../../tools/azure-devtools
 ../../../tools/azure-sdk-tools

eng/tox/install_dev_build_dependency.py

@@ -13,13 +13,11 @@
 from os import path
 from subprocess import check_call
 
-from pip._internal.operations import freeze
-
 # import common_task module
 root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
 common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
 sys.path.append(common_task_path)
-from common_tasks import process_glob_string
+from common_tasks import process_glob_string, get_installed_packages
 from tox_helper_tasks import get_package_details
 
 EXCLUDED_PKGS = [
@@ -34,10 +32,10 @@
 # This script verifies installed package version and ensure all installed pacakges are dev build version
 
 
-def get_installed_packages(pkg_name_to_exclude):
+def get_installed_azure_packages(pkg_name_to_exclude):
     # This method returns a list of installed azure sdk packages
     installed_pkgs = [
-        p.split("==")[0] for p in freeze.freeze() if p.startswith("azure-")
+        p.split("==")[0] for p in get_installed_packages() if p.startswith("azure-")
     ]
 
     # Get valid list of Azure SDK packages in repo
@@ -100,7 +98,7 @@ def install_packages(packages):
 
 def install_dev_build_packages(pkg_name_to_exclude):
     # Uninstall GA version and reinstall dev build version of dependent packages
-    azure_pkgs = get_installed_packages(pkg_name_to_exclude)
+    azure_pkgs = get_installed_azure_packages(pkg_name_to_exclude)
     uninstall_packages(azure_pkgs)
     install_packages(azure_pkgs)
 

eng/tox/verify_installed_packages.py

@@ -9,8 +9,13 @@
 import os
 import sys
 import logging
-from pip._internal.operations import freeze
+from os import path
 
+# import common_task module
+root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
+common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
+sys.path.append(common_task_path)
+from common_tasks import get_installed_packages
 
 def verify_packages(package_file_path):
     # this method verifies packages installed on machine is matching the expected package version
@@ -29,7 +34,7 @@ def verify_packages(package_file_path):
         sys.exit(1)
 
     # find installed and expected packages
-    installed = dict(p.split('==') for p in freeze.freeze() if p.startswith('azure') and "==" in p)
+    installed = dict(p.split('==') for p in get_installed_packages() if p.startswith('azure') and "==" in p)
     expected = dict(p.split('==') for p in packages)
 
     missing_packages = [pkg for pkg in expected.keys() if installed.get(pkg) != expected.get(pkg)]

pylintrc

@@ -3,7 +3,7 @@ ignore-patterns=test_*,conftest,setup
 reports=no
 
 # PYLINT DIRECTORY BLACKLIST.
-ignore=_generated,samples,examples,test,tests,doc,.tox
+ignore=_vendor,_generated,samples,examples,test,tests,doc,.tox
 
 init-hook='import sys; sys.path.insert(0, os.path.abspath(os.getcwd().rsplit("azure-sdk-for-python", 1)[0] + "azure-sdk-for-python/scripts/pylint_custom_plugin"))'
 load-plugins=pylint_guidelines_checker
@@ -14,7 +14,7 @@ load-plugins=pylint_guidelines_checker
 # cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
 # too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
 # Let's black deal with bad-continuation
-disable=useless-object-inheritance,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,too-few-public-methods,bad-continuation,check-docstrings
+disable=useless-object-inheritance,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,too-few-public-methods,bad-continuation,check-docstrings,import-outside-toplevel
 
 [FORMAT]
 max-line-length=120

scripts/devops_tasks/common_tasks.py

@@ -22,7 +22,7 @@
 import pdb
 
 # Assumes the presence of setuptools
-from pkg_resources import parse_version, parse_requirements, Requirement
+from pkg_resources import parse_version, parse_requirements, Requirement, WorkingSet, working_set
 
 # this assumes the presence of "packaging"
 from packaging.specifiers import SpecifierSet
@@ -420,4 +420,15 @@ def find_tools_packages(root_path):
     glob_string = os.path.join(root_path, "tools", "*", "setup.py")
     pkgs = [os.path.basename(os.path.dirname(p)) for p in glob.glob(glob_string)]
     logging.info("Packages in tools: {}".format(pkgs))
-    return pkgs
+    return pkgs
+
+
+def get_installed_packages(paths = None):
+    """Find packages in default or given lib paths
+    """
+    # WorkingSet returns installed packages in given path
+    # working_set returns installed packages in default path
+    # if paths is set then find installed packages from given paths
+    ws = WorkingSet(paths) if paths else working_set  
+    return ["{0}=={1}".format(p.project_name, p.version) for p in ws]
+

scripts/devops_tasks/test_regression.py

@@ -22,10 +22,10 @@
     filter_dev_requirements,
     find_packages_missing_on_pypi,
     find_whl,
-    find_tools_packages
+    find_tools_packages,
+    get_installed_packages
 )
 from git_helper import get_release_tag, git_checkout_tag, git_checkout_branch, clone_repo
-from pip._internal.operations import freeze
 
 AZURE_GLOB_STRING = "azure*"
 
@@ -222,7 +222,7 @@ def _install_packages(self, dependent_pkg_path, pkg_to_exclude):
         temp_dir = self.context.temp_path
 
         list_to_exclude = [pkg_to_exclude,]
-        installed_pkgs = [p.split('==')[0] for p in list(freeze.freeze(paths=self.context.venv.lib_paths)) if p.startswith('azure-')]
+        installed_pkgs = [p.split('==')[0] for p in get_installed_packages(self.context.venv.lib_paths) if p.startswith('azure-')]
         logging.info("Installed azure sdk packages:{}".format(installed_pkgs))
 
         # Do not exclude list of packages in tools directory and so these tools packages will be reinstalled from repo branch we are testing
@@ -257,7 +257,7 @@ def _is_package_installed(self, package, version):
         venv_root = self.context.venv.path
         site_packages = self.context.venv.lib_paths
         logging.info("Searching for packages in :{}".format(site_packages))
-        installed_pkgs = list(freeze.freeze(paths=site_packages))
+        installed_pkgs = get_installed_packages(site_packages)
         logging.info("Installed packages: {}".format(installed_pkgs))
         # Verify installed package version
         # Search for exact version or dev build version of current version.

scripts/pylint_custom_plugin/README.md

@@ -3,7 +3,7 @@
 In order to lint for the guidelines, you must make sure you are using the pylintrc file.
 It is recommended you run pylint at the library package level to be consistent with how the CI runs pylint.  
 
-Check that you are running pylint version >=2.31 and astroid version >=2.25.
+Check that you are running pylint version >=2.5.2 and astroid version >=2.4.1.
 
 **How to run pylint locally using the pylintrc:**