Reorganize test run in automation (Azure#1624)

* Reorganize test run in automation

* Remove unnessary change in the VM test

* Address review comments

.gitignore

@@ -65,6 +65,10 @@ src/build
 # Auxiliary files
 command_coverage.txt
 
-#Test artifacts
+# Test artifacts
 private_config.json
 scripts/smart_create_gen/config.ini
+test_results/
+
+# Code coverage
+.coverage

.travis.yml

@@ -12,11 +12,4 @@ install:
   - pip install -qqq virtualenv # used by package_verify script
   - python scripts/dev_setup.py
 script: 
-  - export PYTHONPATH=$PATHONPATH:./src
-  - python -m azure.cli -h
-  - python ./scripts/run_pylint.py
-  - nosetests -v --processes=-1 --process-timeout=600 -w src/azure-cli/azure/cli/tests
-  - nosetests -v --processes=-1 --process-timeout=600 -w src/azure-cli-core/azure/cli/core/tests
-  - python scripts/command_modules/test.py
-  - sh scripts/package_verify.sh
-  - python scripts/license/verify.py
+  - ./scripts/build_in_travis.sh

requirements.txt

@@ -2,6 +2,7 @@ adal==0.4.3
 applicationinsights==0.10.0
 argcomplete==1.3.0
 colorama==0.3.7
+coverage==4.2
 jmespath
 mock==1.3.0
 nose==1.3.7

scripts/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/coverage/__init__.py

@@ -0,0 +1,7 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+"""Code coverage related automation code"""
+

scripts/automation/coverage/run.py

@@ -0,0 +1,110 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import os.path
+import sys
+import itertools
+
+from coverage import Coverage
+
+import azure.cli.core.application as cli_application
+from ..tests.nose_helper import get_nose_runner
+from ..utilities.path import get_core_modules_paths_with_tests, \
+    get_command_modules_paths_with_tests, get_repo_root, get_test_results_dir, make_dirs
+
+
+# TODO: Fix track command logic in vcr_test_base.py.
+def run_command_coverage(output_file='command_coverage.txt', output_dir=None):
+    class CoverageContext(object):
+        def __enter__(self):
+            os.environ['AZURE_CLI_TEST_TRACK_COMMANDS'] = '1'
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            del os.environ['AZURE_CLI_TEST_TRACK_COMMANDS']
+
+    if output_dir is None:
+        from ..utilities.path import get_test_results_dir
+        output_dir = get_test_results_dir(with_timestamp=True, prefix='cmd_cov')
+
+    coverage_file = os.path.join(output_dir, output_file)
+    if os.path.isfile(coverage_file):
+        os.remove(coverage_file)
+
+    config = cli_application.Configuration([])
+    cli_application.APPLICATION = cli_application.Application(config)
+
+    cmd_table = config.get_command_table()
+    cmd_list = cmd_table.keys()
+    cmd_set = set(cmd_list)
+
+    print('Running tests...')
+    with CoverageContext():
+        test_result = run_all_tests()
+        if not test_result:
+            print("Tests failed")
+            sys.exit(1)
+        else:
+            print('Tests passed.')
+
+    commands_tested_with_params = [line.rstrip('\n') for line in open(coverage_file)]
+
+    commands_tested = []
+    for tested_command in commands_tested_with_params:
+        for c in cmd_list:
+            if tested_command.startswith(c):
+                commands_tested.append(c)
+
+    commands_tested_set = set(commands_tested)
+    untested = list(cmd_set - commands_tested_set)
+    print()
+    print("Untested commands")
+    print("=================")
+    print('\n'.join(sorted(untested)))
+    percentage_tested = (len(commands_tested_set) * 100.0 / len(cmd_set))
+    print()
+    print('Total commands {}, Tested commands {}, Untested commands {}'.format(
+        len(cmd_set),
+        len(commands_tested_set),
+        len(cmd_set) - len(commands_tested_set)))
+    print('COMMAND COVERAGE {0:.2f}%'.format(percentage_tested))
+
+
+class CoverageContext(object):
+    def __init__(self):
+        self._cov = Coverage(cover_pylib=False)
+        self._cov.start()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._cov.stop()
+
+
+def run_code_coverage():
+    # create test results folder
+    test_results_folder = get_test_results_dir(with_timestamp=True, prefix='cover')
+
+    # get test runner
+    run_nose = get_nose_runner(test_results_folder, xunit_report=False, exclude_integration=True,
+                               code_coverage=True, parallel=False)
+
+    # list test modules
+    test_modules = itertools.chain(get_core_modules_paths_with_tests(),
+                                   get_command_modules_paths_with_tests())
+
+    # run code coverage on each project
+    for index, (name, _, test_path) in enumerate(test_modules):
+        with CoverageContext():
+            run_nose(name, test_path)
+
+        import shutil
+        shutil.move('.coverage', os.path.join(test_results_folder, '.coverage.{}'.format(name)))
+
+
+if __name__ == '__main__':
+    run_code_coverage()

scripts/automation/setup/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/style/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/style/run.py

@@ -0,0 +1,28 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os.path
+import multiprocessing
+from subprocess import check_call
+from itertools import chain
+from ..utilities.path import get_command_modules_paths, get_core_modules_paths, get_repo_root
+
+
+def run_pylint():
+    modules_list = ' '.join(os.path.join(path, 'azure') for _, path in
+                            chain(get_command_modules_paths(), get_core_modules_paths()))
+    arguments = '{} --rcfile={} -j {} -r n -d I0013'.format(
+        modules_list,
+        os.path.join(get_repo_root(), 'pylintrc'),
+        multiprocessing.cpu_count())
+
+    print('pylint arguments: ' + arguments)
+
+    check_call(('python -m pylint ' + arguments).split())
+
+    print('Pylint done')
+
+if __name__ == '__main__':
+    run_pylint()

scripts/automation/tests/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/tests/nose_helper.py

@@ -0,0 +1,58 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from datetime import datetime
+
+
+def get_nose_runner(report_folder, parallel=True, process_timeout=600, process_restart=True,
+                    xunit_report=False, exclude_integration=True, code_coverage=False):
+    """Create a nose execution method"""
+
+    def _run_nose(name, working_dir):
+        import nose
+        import os.path
+
+        if not report_folder \
+                or not os.path.exists(report_folder) \
+                or not os.path.isdir(report_folder):
+            raise ValueError('Report folder {} does not exist'.format(report_folder))
+
+        arguments = ['-w', working_dir, '-v']
+        if parallel:
+            arguments += ['--processes=-1', '--process-timeout={}'.format(process_timeout)]
+            if process_restart:
+                arguments += ['--process-restartworker']
+
+        if xunit_report:
+            log_file = os.path.join(report_folder, name + '-report.xml')
+            arguments += ['--with-xunit', '--xunit-file', log_file]
+        else:
+            log_file = None
+
+        if exclude_integration:
+            arguments += ['--ignore-files=integration*']
+
+        if code_coverage:
+            # coverage_config = os.path.join(os.path.dirname(__file__), '.coveragerc')
+            # coverage_folder = os.path.join(report_folder, 'code_coverage')
+            # make_dirs(coverage_folder)
+            # if not os.path.exists(coverage_folder) or not os.path.isdir(coverage_folder):
+            #     raise Exception('Failed to create folder {} for code coverage result'
+            #                     .format(coverage_folder))
+
+            arguments += ['--with-coverage']
+
+        debug_file = os.path.join(report_folder, name + '-debug.log')
+        arguments += ['--debug-log={}'.format(debug_file)]
+
+        print()
+        print('<<< Run {} >>>'.format(name))
+        start = datetime.now()
+        result = nose.run(argv=arguments)
+        end = datetime.now()
+
+        return result, start, end, log_file
+
+    return _run_nose

scripts/automation/tests/run.py

@@ -0,0 +1,44 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import itertools
+import sys
+
+from .nose_helper import get_nose_runner
+from ..utilities.display import print_records
+from ..utilities.path import get_command_modules_paths_with_tests, \
+                             get_core_modules_paths_with_tests,\
+                             get_test_results_dir
+
+
+def run_all():
+    # create test results folder
+    test_results_folder = get_test_results_dir(with_timestamp=True, prefix='tests')
+
+    # get test runner
+    run_nose = get_nose_runner(test_results_folder, xunit_report=True, exclude_integration=True)
+
+    # get test list
+    modules_to_test = itertools.chain(
+        get_core_modules_paths_with_tests(),
+        get_command_modules_paths_with_tests())
+
+    # run tests
+    passed = True
+    module_results = []
+    for name, _, test_path in modules_to_test:
+        result, start, end, log_file = run_nose(name, test_path)
+        passed &= result
+        record = (name, start.strftime('%H:%M:%D'), str((end - start).total_seconds()),
+                  'Pass' if result else 'Fail')
+
+        module_results.append(record)
+
+    print_records(module_results, title='test results')
+
+    return passed
+
+if __name__ == '__main__':
+    sys.exit(0 if run_all() else 1)

scripts/automation/utilities/__init__.py

@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+

scripts/automation/utilities/const.py

@@ -0,0 +1,7 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+
+COMMAND_MODULE_PREFIX = 'azure-cli-'

scripts/automation/utilities/display.py

@@ -0,0 +1,47 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+
+def get_print_format(records):
+    """Find the best format to display the given list of records in table format"""
+    if not records:
+        raise ValueError('missing parameter records')
+
+    if not isinstance(records, list):
+        raise ValueError('records is not a list')
+
+    size = len(records[0])
+    max_len = [0] * size
+
+    col_index = list(range(size))
+    for rec in records:
+        if len(rec) != size:
+            raise ValueError('size of elements in the records set are not equal')
+
+        for i in col_index:
+            max_len[i] = max(max_len[i], len(str(rec[i])))
+
+    recommend_format = ''
+    for each in max_len:
+        recommend_format += '{:' + str(each + 2) + '}'
+
+    return recommend_format, max_len
+
+
+def print_records(records, print_format=None, title=None, foot_notes=None):
+    """Print a list of tuples with a print format."""
+    print_format = print_format or get_print_format(records)[0]
+    if print_format is None:
+        raise ValueError('print format is required')
+
+    print()
+    print("Summary" + ': {}'.format(title) if title is not None else '')
+    print("==========================")
+    for rec in records:
+        print(print_format.format(*rec))
+    print("==========================")
+    if foot_notes:
+        for each in foot_notes:
+            print('* ' + each)