forked from Azure/azure-cli
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reorganize test run in automation (Azure#1624)
* Reorganize test run in automation * Remove unnessary change in the VM test * Address review comments
- Loading branch information
1 parent
6206ea6
commit f564c90
Showing
26 changed files
with
474 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
"""Code coverage related automation code""" | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
import os | ||
import os.path | ||
import sys | ||
import itertools | ||
|
||
from coverage import Coverage | ||
|
||
import azure.cli.core.application as cli_application | ||
from ..tests.nose_helper import get_nose_runner | ||
from ..utilities.path import get_core_modules_paths_with_tests, \ | ||
get_command_modules_paths_with_tests, get_repo_root, get_test_results_dir, make_dirs | ||
|
||
|
||
# TODO: Fix track command logic in vcr_test_base.py. | ||
def run_command_coverage(output_file='command_coverage.txt', output_dir=None): | ||
class CoverageContext(object): | ||
def __enter__(self): | ||
os.environ['AZURE_CLI_TEST_TRACK_COMMANDS'] = '1' | ||
return self | ||
|
||
def __exit__(self, exc_type, exc_val, exc_tb): | ||
del os.environ['AZURE_CLI_TEST_TRACK_COMMANDS'] | ||
|
||
if output_dir is None: | ||
from ..utilities.path import get_test_results_dir | ||
output_dir = get_test_results_dir(with_timestamp=True, prefix='cmd_cov') | ||
|
||
coverage_file = os.path.join(output_dir, output_file) | ||
if os.path.isfile(coverage_file): | ||
os.remove(coverage_file) | ||
|
||
config = cli_application.Configuration([]) | ||
cli_application.APPLICATION = cli_application.Application(config) | ||
|
||
cmd_table = config.get_command_table() | ||
cmd_list = cmd_table.keys() | ||
cmd_set = set(cmd_list) | ||
|
||
print('Running tests...') | ||
with CoverageContext(): | ||
test_result = run_all_tests() | ||
if not test_result: | ||
print("Tests failed") | ||
sys.exit(1) | ||
else: | ||
print('Tests passed.') | ||
|
||
commands_tested_with_params = [line.rstrip('\n') for line in open(coverage_file)] | ||
|
||
commands_tested = [] | ||
for tested_command in commands_tested_with_params: | ||
for c in cmd_list: | ||
if tested_command.startswith(c): | ||
commands_tested.append(c) | ||
|
||
commands_tested_set = set(commands_tested) | ||
untested = list(cmd_set - commands_tested_set) | ||
print() | ||
print("Untested commands") | ||
print("=================") | ||
print('\n'.join(sorted(untested))) | ||
percentage_tested = (len(commands_tested_set) * 100.0 / len(cmd_set)) | ||
print() | ||
print('Total commands {}, Tested commands {}, Untested commands {}'.format( | ||
len(cmd_set), | ||
len(commands_tested_set), | ||
len(cmd_set) - len(commands_tested_set))) | ||
print('COMMAND COVERAGE {0:.2f}%'.format(percentage_tested)) | ||
|
||
|
||
class CoverageContext(object): | ||
def __init__(self): | ||
self._cov = Coverage(cover_pylib=False) | ||
self._cov.start() | ||
|
||
def __enter__(self): | ||
return self | ||
|
||
def __exit__(self, exc_type, exc_val, exc_tb): | ||
self._cov.stop() | ||
|
||
|
||
def run_code_coverage(): | ||
# create test results folder | ||
test_results_folder = get_test_results_dir(with_timestamp=True, prefix='cover') | ||
|
||
# get test runner | ||
run_nose = get_nose_runner(test_results_folder, xunit_report=False, exclude_integration=True, | ||
code_coverage=True, parallel=False) | ||
|
||
# list test modules | ||
test_modules = itertools.chain(get_core_modules_paths_with_tests(), | ||
get_command_modules_paths_with_tests()) | ||
|
||
# run code coverage on each project | ||
for index, (name, _, test_path) in enumerate(test_modules): | ||
with CoverageContext(): | ||
run_nose(name, test_path) | ||
|
||
import shutil | ||
shutil.move('.coverage', os.path.join(test_results_folder, '.coverage.{}'.format(name))) | ||
|
||
|
||
if __name__ == '__main__': | ||
run_code_coverage() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
import os.path | ||
import multiprocessing | ||
from subprocess import check_call | ||
from itertools import chain | ||
from ..utilities.path import get_command_modules_paths, get_core_modules_paths, get_repo_root | ||
|
||
|
||
def run_pylint(): | ||
modules_list = ' '.join(os.path.join(path, 'azure') for _, path in | ||
chain(get_command_modules_paths(), get_core_modules_paths())) | ||
arguments = '{} --rcfile={} -j {} -r n -d I0013'.format( | ||
modules_list, | ||
os.path.join(get_repo_root(), 'pylintrc'), | ||
multiprocessing.cpu_count()) | ||
|
||
print('pylint arguments: ' + arguments) | ||
|
||
check_call(('python -m pylint ' + arguments).split()) | ||
|
||
print('Pylint done') | ||
|
||
if __name__ == '__main__': | ||
run_pylint() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
from datetime import datetime | ||
|
||
|
||
def get_nose_runner(report_folder, parallel=True, process_timeout=600, process_restart=True, | ||
xunit_report=False, exclude_integration=True, code_coverage=False): | ||
"""Create a nose execution method""" | ||
|
||
def _run_nose(name, working_dir): | ||
import nose | ||
import os.path | ||
|
||
if not report_folder \ | ||
or not os.path.exists(report_folder) \ | ||
or not os.path.isdir(report_folder): | ||
raise ValueError('Report folder {} does not exist'.format(report_folder)) | ||
|
||
arguments = ['-w', working_dir, '-v'] | ||
if parallel: | ||
arguments += ['--processes=-1', '--process-timeout={}'.format(process_timeout)] | ||
if process_restart: | ||
arguments += ['--process-restartworker'] | ||
|
||
if xunit_report: | ||
log_file = os.path.join(report_folder, name + '-report.xml') | ||
arguments += ['--with-xunit', '--xunit-file', log_file] | ||
else: | ||
log_file = None | ||
|
||
if exclude_integration: | ||
arguments += ['--ignore-files=integration*'] | ||
|
||
if code_coverage: | ||
# coverage_config = os.path.join(os.path.dirname(__file__), '.coveragerc') | ||
# coverage_folder = os.path.join(report_folder, 'code_coverage') | ||
# make_dirs(coverage_folder) | ||
# if not os.path.exists(coverage_folder) or not os.path.isdir(coverage_folder): | ||
# raise Exception('Failed to create folder {} for code coverage result' | ||
# .format(coverage_folder)) | ||
|
||
arguments += ['--with-coverage'] | ||
|
||
debug_file = os.path.join(report_folder, name + '-debug.log') | ||
arguments += ['--debug-log={}'.format(debug_file)] | ||
|
||
print() | ||
print('<<< Run {} >>>'.format(name)) | ||
start = datetime.now() | ||
result = nose.run(argv=arguments) | ||
end = datetime.now() | ||
|
||
return result, start, end, log_file | ||
|
||
return _run_nose |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
import itertools | ||
import sys | ||
|
||
from .nose_helper import get_nose_runner | ||
from ..utilities.display import print_records | ||
from ..utilities.path import get_command_modules_paths_with_tests, \ | ||
get_core_modules_paths_with_tests,\ | ||
get_test_results_dir | ||
|
||
|
||
def run_all(): | ||
# create test results folder | ||
test_results_folder = get_test_results_dir(with_timestamp=True, prefix='tests') | ||
|
||
# get test runner | ||
run_nose = get_nose_runner(test_results_folder, xunit_report=True, exclude_integration=True) | ||
|
||
# get test list | ||
modules_to_test = itertools.chain( | ||
get_core_modules_paths_with_tests(), | ||
get_command_modules_paths_with_tests()) | ||
|
||
# run tests | ||
passed = True | ||
module_results = [] | ||
for name, _, test_path in modules_to_test: | ||
result, start, end, log_file = run_nose(name, test_path) | ||
passed &= result | ||
record = (name, start.strftime('%H:%M:%D'), str((end - start).total_seconds()), | ||
'Pass' if result else 'Fail') | ||
|
||
module_results.append(record) | ||
|
||
print_records(module_results, title='test results') | ||
|
||
return passed | ||
|
||
if __name__ == '__main__': | ||
sys.exit(0 if run_all() else 1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
|
||
COMMAND_MODULE_PREFIX = 'azure-cli-' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# -------------------------------------------------------------------------------------------- | ||
# Copyright (c) Microsoft Corporation. All rights reserved. | ||
# Licensed under the MIT License. See License.txt in the project root for license information. | ||
# -------------------------------------------------------------------------------------------- | ||
|
||
|
||
def get_print_format(records): | ||
"""Find the best format to display the given list of records in table format""" | ||
if not records: | ||
raise ValueError('missing parameter records') | ||
|
||
if not isinstance(records, list): | ||
raise ValueError('records is not a list') | ||
|
||
size = len(records[0]) | ||
max_len = [0] * size | ||
|
||
col_index = list(range(size)) | ||
for rec in records: | ||
if len(rec) != size: | ||
raise ValueError('size of elements in the records set are not equal') | ||
|
||
for i in col_index: | ||
max_len[i] = max(max_len[i], len(str(rec[i]))) | ||
|
||
recommend_format = '' | ||
for each in max_len: | ||
recommend_format += '{:' + str(each + 2) + '}' | ||
|
||
return recommend_format, max_len | ||
|
||
|
||
def print_records(records, print_format=None, title=None, foot_notes=None): | ||
"""Print a list of tuples with a print format.""" | ||
print_format = print_format or get_print_format(records)[0] | ||
if print_format is None: | ||
raise ValueError('print format is required') | ||
|
||
print() | ||
print("Summary" + ': {}'.format(title) if title is not None else '') | ||
print("==========================") | ||
for rec in records: | ||
print(print_format.format(*rec)) | ||
print("==========================") | ||
if foot_notes: | ||
for each in foot_notes: | ||
print('* ' + each) |
Oops, something went wrong.