{CI} Optimize the extension test: Use azdev test for testing, consi…

…stent with the main repo (Azure#5925)

* refine extension test

* Update test_source.py

* update

.azure-pipelines/templates/azdev_setup.yml

@@ -0,0 +1,25 @@
+parameters:
+  - name: CLIRepoPath
+    type: string
+    default: '../azure-cli'
+  - name: CLIExtensionRepoPath
+    type: string
+    default: './'
+steps:
+  - bash: |
+      set -ev
+      python -m venv env
+      chmod +x env/bin/activate
+      source ./env/bin/activate
+      # clone azure-cli
+      git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+      python -m pip install -U pip
+      pip install azdev
+      azdev --version
+      azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug
+      pip list -v
+      az --version
+    displayName: 'azdev setup'
+    env:
+      CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}
+      CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }}

azure-pipelines.yml

@@ -60,26 +60,11 @@ jobs:
     displayName: 'Use Python 3.10'
     inputs:
       versionSpec: 3.10
+  - template: .azure-pipelines/templates/azdev_setup.yml
   - bash: |
+      #!/usr/bin/env bash
       set -ev
-
-      # prepare and activate virtualenv
-      python -m venv env/
-
-      chmod +x ./env/bin/activate
       source ./env/bin/activate
-
-      # clone azure-cli
-      git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
-
-      python -m pip install -U pip
-      pip install -q azdev
-
-      azdev setup -c ../azure-cli -r ./
-
-      azdev --version
-      az --version
-
       azdev verify license
 
 - job: StaticAnalysis
@@ -130,9 +115,15 @@ jobs:
       displayName: 'Use Python $(python.version)'
       inputs:
         versionSpec: '$(python.version)'
+    - template: .azure-pipelines/templates/azdev_setup.yml
     - bash: pip install wheel==0.30.0
       displayName: 'Install wheel==0.30.0'
-    - bash: ./scripts/ci/test_source.sh
+    - bash: |
+        #!/usr/bin/env bash
+        set -ev
+        source ./env/bin/activate
+        az --version
+        python scripts/ci/test_source.py -v
       displayName: 'Run integration test and build test'
       env:
         ADO_PULL_REQUEST_LATEST_COMMIT: HEAD
@@ -148,24 +139,11 @@ jobs:
       displayName: 'Use Python 3.10'
       inputs:
         versionSpec: 3.10
+    - template: .azure-pipelines/templates/azdev_setup.yml
     - bash: |
+        #!/usr/bin/env bash
         set -ev
-
-        # prepare and activate virtualenv
-        pip install virtualenv
-        python -m virtualenv venv/
-        source ./venv/bin/activate
-
-        # clone azure-cli
-        git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
-
-        python -m pip install -U pip
-        pip install azdev
-
-        azdev --version
-
-        azdev setup -c ../azure-cli -r ./
-
+        source ./env/bin/activate
         # overwrite the default AZURE_EXTENSION_DIR set by ADO
         AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version
 

scripts/ci/test_source.py

@@ -11,15 +11,10 @@
 import os
 import sys
 import tempfile
-import unittest
 import shutil
 import shlex
-from subprocess import check_output, check_call, CalledProcessError
-
-from unittest import mock
-from wheel.install import WHEEL_INFO_RE
-from six import with_metaclass
 
+from subprocess import check_output, CalledProcessError, run
 from util import SRC_PATH
 
 logger = logging.getLogger(__name__)
@@ -62,56 +57,42 @@
                f'ALL_TESTS: {ALL_TESTS}.')
 
 
-class TestExtensionSourceMeta(type):
-    def __new__(mcs, name, bases, _dict):
-
-        def gen_test(ext_path):
-            def test(self):
-                ext_install_dir = os.path.join(self.ext_dir, 'ext')
-                pip_args = [sys.executable, '-m', 'pip', 'install', '--upgrade', '--target',
-                            ext_install_dir, ext_path]
-                check_call(pip_args)
-                unittest_args = [sys.executable, '-m', 'unittest', 'discover', '-v', ext_path]
-                env = os.environ.copy()
-                env['PYTHONPATH'] = ext_install_dir
-                env['AZURE_CORE_USE_COMMAND_INDEX'] = 'false'
-                check_call(unittest_args, env=env)
-            return test
-
-        for tname, ext_path in ALL_TESTS:
-            test_name = "test_%s" % tname
-            _dict[test_name] = gen_test(ext_path)
-        return type.__new__(mcs, name, bases, _dict)
-
-
-class TestExtensionSource(with_metaclass(TestExtensionSourceMeta, unittest.TestCase)):
-
-    def setUp(self):
-        self.ext_dir = tempfile.mkdtemp()
-        self.mock_env = mock.patch.dict(os.environ, {'AZURE_EXTENSION_DIR': self.ext_dir})
-        self.mock_env.start()
-
-    def tearDown(self):
-        self.mock_env.stop()
-        shutil.rmtree(self.ext_dir)
-
-
-class TestSourceWheels(unittest.TestCase):
-
-    def test_source_wheels(self):
-        # Test we can build all sources into wheels and that metadata from the wheel is valid
-        built_whl_dir = tempfile.mkdtemp()
-        source_extensions = [os.path.join(SRC_PATH, n) for n in os.listdir(SRC_PATH)
-                             if os.path.isdir(os.path.join(SRC_PATH, n))]
-        for s in source_extensions:
-            if not os.path.isfile(os.path.join(s, 'setup.py')):
-                continue
-            try:
-                check_output(['python', 'setup.py', 'bdist_wheel', '-q', '-d', built_whl_dir], cwd=s)
-            except CalledProcessError as err:
-                self.fail("Unable to build extension {} : {}".format(s, err))
-        shutil.rmtree(built_whl_dir)
+def run_command(cmd, check_return_code=False, cwd=None):
+    logger.info(f'cmd: {cmd}')
+    out = run(cmd, check=True, cwd=cwd)
+    if check_return_code and out.returncode:
+        raise RuntimeError(f"{cmd} failed")
+
+
+def test_extension():
+    for tname, ext_path in ALL_TESTS:
+        ext_name = ext_path.split('/')[-1]
+        logger.info(f'installing extension: {ext_name}')
+        cmd = ['azdev', 'extension', 'add', ext_name]
+        run_command(cmd, check_return_code=True)
+        test_args = [sys.executable, '-m', 'azdev', 'test', '--no-exitfirst', '--discover', '--verbose', ext_name]
+        logger.warning(f'test_args: {test_args}')
+        run_command(test_args, check_return_code=True)
+        logger.info(f'uninstalling extension: {ext_name}')
+        cmd = ['azdev', 'extension', 'remove', ext_name]
+        run_command(cmd, check_return_code=True)
+
+
+def test_source_wheels():
+    # Test we can build all sources into wheels and that metadata from the wheel is valid
+    built_whl_dir = tempfile.mkdtemp()
+    source_extensions = [os.path.join(SRC_PATH, n) for n in os.listdir(SRC_PATH)
+                         if os.path.isdir(os.path.join(SRC_PATH, n))]
+    for s in source_extensions:
+        if not os.path.isfile(os.path.join(s, 'setup.py')):
+            continue
+        try:
+            check_output(['python', 'setup.py', 'bdist_wheel', '-q', '-d', built_whl_dir], cwd=s)
+        except CalledProcessError as err:
+            raise("Unable to build extension {} : {}".format(s, err))
+    shutil.rmtree(built_whl_dir)
 
 
 if __name__ == '__main__':
-    unittest.main()
+    test_extension()
+    test_source_wheels()