From 24e3f934f9029454f9f84be9f0e58eb5856e8b3a Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 20 Aug 2017 09:22:44 -0700 Subject: [PATCH] Overhaul Travis build configuration 1. Adopt build stage feature. Separate the build process into build, verify and publish stages. 2. Move build utilties from scripts folder to its own tools folder for better organization. 3. Publish wheel artifacts to a blob storage container for long-term persistence. --- .travis.yml | 58 ++++--- pylintrc | 2 +- requirements.txt | 1 - scripts/{batch => ci}/app/install.sh | 0 scripts/{batch => ci}/app/run_test.sh | 0 scripts/{batch => ci}/app/schedule.py | 0 scripts/{batch => ci}/app/schedule.sh | 0 scripts/ci/artifacts.sh | 4 + scripts/ci/build.sh | 160 ++++++++++++++++++ scripts/ci/publish.sh | 30 ++++ scripts/ci/test_automation.sh | 17 ++ scripts/ci/test_integration.sh | 15 ++ scripts/ci/test_static.sh | 37 ++++ scripts/dev_setup.py | 2 +- scripts/license/_common.py | 43 ----- scripts/license/add.py | 20 --- scripts/license/verify.py | 21 --- scripts/package_verify.sh | 2 +- src/__init__.py | 1 - src/command_modules/__init__.py | 1 - {scripts => tools}/automation/__init__.py | 0 tools/automation/__main__.py | 19 +++ .../automation/commandlint/__init__.py | 0 .../automation/commandlint/allowed-error.json | 0 .../automation/commandlint/run.py | 0 .../automation/coverage/__init__.py | 0 {scripts => tools}/automation/coverage/run.py | 0 .../automation/release/README.md | 0 .../automation/release/__init__.py | 0 .../automation/release/check.py | 0 .../automation/release/notes.py | 0 .../automation/release/packaged.py | 0 {scripts => tools}/automation/release/run.py | 0 .../automation/release/version_patcher.py | 0 .../automation/setup/__init__.py | 0 .../automation/setup/install_modules.py | 0 .../automation/style/__init__.py | 0 {scripts => tools}/automation/style/pep8.py | 0 .../automation/style/pylint_disable_check.py | 0 {scripts => tools}/automation/style/run.py | 4 +- .../automation/tests/__init__.py | 0 .../automation/tests/check_vcr_recordings.py | 0 .../automation/tests/nose_helper.py | 0 {scripts => tools}/automation/tests/run.py | 21 ++- .../automation/tests/verify_dependencies.py | 0 .../automation/tests/verify_packages.py | 92 ++++------ .../automation/tests/verify_readme_history.py | 23 +-- .../automation/utilities/__init__.py | 0 .../automation/utilities/const.py | 0 .../automation/utilities/display.py | 0 .../automation/utilities/path.py | 0 tools/automation/verify/__init__.py | 57 +++++++ .../automation/verify/default_modules.py | 50 +++--- .../automation/verify/doc_source_map.py | 14 +- tools/scripts/azdev | 8 + tools/scripts/azdev.bat | 9 + {scripts => tools/scripts}/check_style | 0 {scripts => tools/scripts}/check_style.bat | 0 {scripts => tools/scripts}/run_tests | 0 {scripts => tools/scripts}/run_tests.bat | 2 +- {scripts => tools}/setup.py | 17 +- 61 files changed, 496 insertions(+), 234 deletions(-) rename scripts/{batch => ci}/app/install.sh (100%) rename scripts/{batch => ci}/app/run_test.sh (100%) rename scripts/{batch => ci}/app/schedule.py (100%) rename scripts/{batch => ci}/app/schedule.sh (100%) create mode 100644 scripts/ci/artifacts.sh create mode 100755 scripts/ci/build.sh create mode 100755 scripts/ci/publish.sh create mode 100755 scripts/ci/test_automation.sh create mode 100755 scripts/ci/test_integration.sh create mode 100755 scripts/ci/test_static.sh delete mode 100644 scripts/license/_common.py delete mode 100644 scripts/license/add.py delete mode 100644 scripts/license/verify.py rename {scripts => tools}/automation/__init__.py (100%) create mode 100644 tools/automation/__main__.py rename {scripts => tools}/automation/commandlint/__init__.py (100%) rename {scripts => tools}/automation/commandlint/allowed-error.json (100%) rename {scripts => tools}/automation/commandlint/run.py (100%) rename {scripts => tools}/automation/coverage/__init__.py (100%) rename {scripts => tools}/automation/coverage/run.py (100%) rename {scripts => tools}/automation/release/README.md (100%) rename {scripts => tools}/automation/release/__init__.py (100%) rename {scripts => tools}/automation/release/check.py (100%) rename {scripts => tools}/automation/release/notes.py (100%) rename {scripts => tools}/automation/release/packaged.py (100%) rename {scripts => tools}/automation/release/run.py (100%) rename {scripts => tools}/automation/release/version_patcher.py (100%) rename {scripts => tools}/automation/setup/__init__.py (100%) rename {scripts => tools}/automation/setup/install_modules.py (100%) rename {scripts => tools}/automation/style/__init__.py (100%) rename {scripts => tools}/automation/style/pep8.py (100%) rename {scripts => tools}/automation/style/pylint_disable_check.py (100%) rename {scripts => tools}/automation/style/run.py (96%) rename {scripts => tools}/automation/tests/__init__.py (100%) rename {scripts => tools}/automation/tests/check_vcr_recordings.py (100%) rename {scripts => tools}/automation/tests/nose_helper.py (100%) rename {scripts => tools}/automation/tests/run.py (84%) rename {scripts => tools}/automation/tests/verify_dependencies.py (100%) rename {scripts => tools}/automation/tests/verify_packages.py (66%) rename {scripts => tools}/automation/tests/verify_readme_history.py (87%) rename {scripts => tools}/automation/utilities/__init__.py (100%) rename {scripts => tools}/automation/utilities/const.py (100%) rename {scripts => tools}/automation/utilities/display.py (100%) rename {scripts => tools}/automation/utilities/path.py (100%) create mode 100644 tools/automation/verify/__init__.py rename scripts/automation/tests/verify_default_modules.py => tools/automation/verify/default_modules.py (60%) rename scripts/automation/tests/verify_doc_source_map.py => tools/automation/verify/doc_source_map.py (89%) create mode 100755 tools/scripts/azdev create mode 100644 tools/scripts/azdev.bat rename {scripts => tools/scripts}/check_style (100%) rename {scripts => tools/scripts}/check_style.bat (100%) rename {scripts => tools/scripts}/run_tests (100%) rename {scripts => tools/scripts}/run_tests.bat (85%) rename {scripts => tools}/setup.py (83%) diff --git a/.travis.yml b/.travis.yml index 5fdff755127..55a613ab951 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,37 @@ -sudo: false +dist: trusty +sudo: off language: python +addons: + apt: + packages: + - libssl-dev + - libffi-dev + - python-dev +install: echo 'skip' +cache: pip jobs: include: - - stage: all - env: CODE_COVERAGE="True" - python: 2.7 - install: python scripts/dev_setup.py - script: scripts/build.sh - - - env: CODE_COVERAGE="True" - python: 3.5 - install: python scripts/dev_setup.py - script: scripts/build.sh - - - env: CODE_COVERAGE="True" - python: 3.6 - install: python scripts/dev_setup.py - script: scripts/build.sh - - - python: 2.7 - install: pip install -qqq virtualenv - script: scripts/package_verify.sh - - - python: 3.6 - install: pip install -qqq virtualenv - script: scripts/package_verify.sh \ No newline at end of file + - stage: build + script: ./scripts/ci/build.sh + - stage: verify + env: PURPOSE='Automation' + script: ./scripts/ci/test_automation.sh + python: 3.6 + - stage: verify + env: PURPOSE='Automation' + script: ./scripts/ci/test_automation.sh + python: 2.7 + - stage: verify + script: ./scripts/ci/test_static.sh + env: PURPOSE='Static Check' + python: 3.6 + - stage: verify + script: ./scripts/ci/test_integration.sh + env: PURPOSE='Integration' + python: 3.6 + - stage: verify + script: ./scripts/ci/test_integration.sh + env: PURPOSE='Integration' + python: 2.7 + - stage: publish + script: ./scripts/ci/publish.sh diff --git a/pylintrc b/pylintrc index a7e2e068958..d28df4238de 100644 --- a/pylintrc +++ b/pylintrc @@ -7,7 +7,7 @@ reports=no # locally-disabled: Warning locally suppressed using disable-msg # cyclic-import: because of https://github.com/PyCQA/pylint/issues/850 # too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods. -disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name +disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code [FORMAT] max-line-length=120 diff --git a/requirements.txt b/requirements.txt index 5149e177034..b0af77de41e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ mock==1.3.0 paramiko==2.0.2 pip==9.0.1 pygments==2.1.3 -pylint==1.7.1 pyOpenSSL==16.2.0 pyyaml==3.11 requests==2.9.1 diff --git a/scripts/batch/app/install.sh b/scripts/ci/app/install.sh similarity index 100% rename from scripts/batch/app/install.sh rename to scripts/ci/app/install.sh diff --git a/scripts/batch/app/run_test.sh b/scripts/ci/app/run_test.sh similarity index 100% rename from scripts/batch/app/run_test.sh rename to scripts/ci/app/run_test.sh diff --git a/scripts/batch/app/schedule.py b/scripts/ci/app/schedule.py similarity index 100% rename from scripts/batch/app/schedule.py rename to scripts/ci/app/schedule.py diff --git a/scripts/batch/app/schedule.sh b/scripts/ci/app/schedule.sh similarity index 100% rename from scripts/batch/app/schedule.sh rename to scripts/ci/app/schedule.sh diff --git a/scripts/ci/artifacts.sh b/scripts/ci/artifacts.sh new file mode 100644 index 00000000000..94e07928e84 --- /dev/null +++ b/scripts/ci/artifacts.sh @@ -0,0 +1,4 @@ +# Build wheel and set the $share_folder to the artifacts folder + +. $(cd $(dirname $0); pwd)/build.sh +share_folder=$(cd $(dirname $0); cd ../../artifacts; pwd) \ No newline at end of file diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh new file mode 100755 index 00000000000..95ca97f168d --- /dev/null +++ b/scripts/ci/build.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +# Build wheel packages containing both CLI product and tests. The script doesn't rely on a pre-existing virtual +# environment. + +set -e + +############################################## +# clean up +mkdir -p ./artifacts +echo `git rev-parse --verify HEAD` > ./artifacts/build.sha + +mkdir -p ./artifacts/build +mkdir -p ./artifacts/app +mkdir -p ./artifacts/testsrc + +output_dir=$(cd artifacts/build && pwd) +testsrc_dir=$(cd artifacts/testsrc && pwd) +app_dir=$(cd artifacts/app && pwd) + +############################################## +# Copy app scripts for batch run - To be retired in the future +cp $(cd $(dirname $0); pwd)/app/* $app_dir + +############################################## +# List tests to the test file - To be replaced by other mechanism in the fucture +# content=`nosetests azure.cli -v --collect-only 2>&1` + +# echo $content > $app_dir/raw.txt +# manifest="$app_dir/all_tests.txt" +# if [ -e $manifest ]; then rm $manifest; fi + +# while IFS='' read -r line || [[ -n "$line" ]]; do +# if [ -n "$line" ] && [ "OK" != "$line" ] && [ "${line:0:3}" != "Ran" ] && [ "${line:0:3}" != "---" ]; then +# parts=($line) +# test_method=${parts[0]} +# test_class=${parts[1]} +# test_class=${test_class##(} +# test_class=${test_class%%)} + +# echo "$test_class $test_method" >> $manifest +# fi +# done <<< "$content" + +############################################## +# build product packages +echo 'Build Azure CLI and its command modules' +for setup_file in $(find src -name 'setup.py'); do + pushd $(dirname $setup_file) + echo "" + echo "Building module at $(pwd) ..." + python setup.py bdist_wheel -d $output_dir sdist -d $output_dir + popd +done + +############################################## +# build test packages +echo 'Build Azure CLI tests package' + +echo "Copy test source code into $build_src ..." +for test_src in $(find src/command_modules -name tests -type d); do + rel_path=${test_src##src/command_modules/} + rel_path=(${rel_path/\// }) + rel_path=${rel_path[1]} + + mkdir -p $testsrc_dir/$rel_path + cp -R $test_src/* $testsrc_dir/$rel_path +done + +for test_src in $(find src -name tests | grep -v command_modules); do + rel_path=${test_src##src/} + rel_path=(${rel_path/\// }) + rel_path=${rel_path[1]} + + mkdir -p $testsrc_dir/$rel_path + cp -R $test_src/* $testsrc_dir/$rel_path +done + +cat >$testsrc_dir/setup.py <>$testsrc_dir/setup.py + fi +done + + +cat >>$testsrc_dir/setup.py < 0 and not contains_header(file_text): - files_without_header.append((cur_file_path, file_text)) - return files_without_header diff --git a/scripts/license/add.py b/scripts/license/add.py deleted file mode 100644 index 8b684b410b9..00000000000 --- a/scripts/license/add.py +++ /dev/null @@ -1,20 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# Add license header to every *.py file in the repo. Can be run multiple times without duplicating the headers. - -from _common import PY_LICENSE_HEADER, get_files_without_header, has_shebang - -files_without_header = get_files_without_header() - -for file_path, file_contents in files_without_header: - with open(file_path, 'w') as modified: - if has_shebang(file_contents): - lines = file_contents.split('\n') - modified.write(lines[0]) - modified.write('\n' + PY_LICENSE_HEADER) - modified.write('\n'.join(lines[1:])) - else: - modified.write(PY_LICENSE_HEADER + file_contents) diff --git a/scripts/license/verify.py b/scripts/license/verify.py deleted file mode 100644 index bb91b022fdd..00000000000 --- a/scripts/license/verify.py +++ /dev/null @@ -1,21 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# Verify that all *.py files have a license header in the file. - -from __future__ import print_function -import sys - -from _common import get_files_without_header - -files_without_header = [file_path for file_path, file_contents in get_files_without_header() if not file_path.endswith('azure_bdist_wheel.py')] - -if files_without_header: - print("Error: The following files don't have the required license headers:", file=sys.stderr) - print('\n'.join(files_without_header), file=sys.stderr) - print("Error: {} file(s) found without license headers.".format(len(files_without_header)), file=sys.stderr) - sys.exit(1) -else: - print('OK') diff --git a/scripts/package_verify.sh b/scripts/package_verify.sh index 1d5d20d9d87..2886d614985 100755 --- a/scripts/package_verify.sh +++ b/scripts/package_verify.sh @@ -3,7 +3,7 @@ set -ex export PYTHONPATH= virtualenv package-verify-env . package-verify-env/bin/activate -pip install -e scripts +pip install -e tools python -m automation.tests.verify_packages python -m automation.tests.verify_dependencies deactivate diff --git a/src/__init__.py b/src/__init__.py index 8dd7ea066b4..34913fb394d 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -2,4 +2,3 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - diff --git a/src/command_modules/__init__.py b/src/command_modules/__init__.py index 8dd7ea066b4..34913fb394d 100644 --- a/src/command_modules/__init__.py +++ b/src/command_modules/__init__.py @@ -2,4 +2,3 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - diff --git a/scripts/automation/__init__.py b/tools/automation/__init__.py similarity index 100% rename from scripts/automation/__init__.py rename to tools/automation/__init__.py diff --git a/tools/automation/__main__.py b/tools/automation/__main__.py new file mode 100644 index 00000000000..225111a6a85 --- /dev/null +++ b/tools/automation/__main__.py @@ -0,0 +1,19 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import argparse +import sys +import automation.verify + +parser = argparse.ArgumentParser(prog='Azure CLI build tools') + +sub_parser = parser.add_subparsers(title='azure cli tools sub commands') +automation.verify.init_args(sub_parser) + +if sys.argv[1:]: + args = parser.parse_args() + args.func(args) +else: + parser.print_help() diff --git a/scripts/automation/commandlint/__init__.py b/tools/automation/commandlint/__init__.py similarity index 100% rename from scripts/automation/commandlint/__init__.py rename to tools/automation/commandlint/__init__.py diff --git a/scripts/automation/commandlint/allowed-error.json b/tools/automation/commandlint/allowed-error.json similarity index 100% rename from scripts/automation/commandlint/allowed-error.json rename to tools/automation/commandlint/allowed-error.json diff --git a/scripts/automation/commandlint/run.py b/tools/automation/commandlint/run.py similarity index 100% rename from scripts/automation/commandlint/run.py rename to tools/automation/commandlint/run.py diff --git a/scripts/automation/coverage/__init__.py b/tools/automation/coverage/__init__.py similarity index 100% rename from scripts/automation/coverage/__init__.py rename to tools/automation/coverage/__init__.py diff --git a/scripts/automation/coverage/run.py b/tools/automation/coverage/run.py similarity index 100% rename from scripts/automation/coverage/run.py rename to tools/automation/coverage/run.py diff --git a/scripts/automation/release/README.md b/tools/automation/release/README.md similarity index 100% rename from scripts/automation/release/README.md rename to tools/automation/release/README.md diff --git a/scripts/automation/release/__init__.py b/tools/automation/release/__init__.py similarity index 100% rename from scripts/automation/release/__init__.py rename to tools/automation/release/__init__.py diff --git a/scripts/automation/release/check.py b/tools/automation/release/check.py similarity index 100% rename from scripts/automation/release/check.py rename to tools/automation/release/check.py diff --git a/scripts/automation/release/notes.py b/tools/automation/release/notes.py similarity index 100% rename from scripts/automation/release/notes.py rename to tools/automation/release/notes.py diff --git a/scripts/automation/release/packaged.py b/tools/automation/release/packaged.py similarity index 100% rename from scripts/automation/release/packaged.py rename to tools/automation/release/packaged.py diff --git a/scripts/automation/release/run.py b/tools/automation/release/run.py similarity index 100% rename from scripts/automation/release/run.py rename to tools/automation/release/run.py diff --git a/scripts/automation/release/version_patcher.py b/tools/automation/release/version_patcher.py similarity index 100% rename from scripts/automation/release/version_patcher.py rename to tools/automation/release/version_patcher.py diff --git a/scripts/automation/setup/__init__.py b/tools/automation/setup/__init__.py similarity index 100% rename from scripts/automation/setup/__init__.py rename to tools/automation/setup/__init__.py diff --git a/scripts/automation/setup/install_modules.py b/tools/automation/setup/install_modules.py similarity index 100% rename from scripts/automation/setup/install_modules.py rename to tools/automation/setup/install_modules.py diff --git a/scripts/automation/style/__init__.py b/tools/automation/style/__init__.py similarity index 100% rename from scripts/automation/style/__init__.py rename to tools/automation/style/__init__.py diff --git a/scripts/automation/style/pep8.py b/tools/automation/style/pep8.py similarity index 100% rename from scripts/automation/style/pep8.py rename to tools/automation/style/pep8.py diff --git a/scripts/automation/style/pylint_disable_check.py b/tools/automation/style/pylint_disable_check.py similarity index 100% rename from scripts/automation/style/pylint_disable_check.py rename to tools/automation/style/pylint_disable_check.py diff --git a/scripts/automation/style/run.py b/tools/automation/style/run.py similarity index 96% rename from scripts/automation/style/run.py rename to tools/automation/style/run.py index 6cde92271b3..b7e91d33898 100644 --- a/scripts/automation/style/run.py +++ b/tools/automation/style/run.py @@ -80,8 +80,8 @@ def run_pep8(modules): parser.print_help() sys.exit(1) - if not args.suites or not any(args.suites): - return_code_sum = run_pylint(selected_modules) + if not args.suites: + return_code_sum = run_pylint(selected_modules) + run_pep8(selected_modules) else: return_code_sum = 0 if 'pep8' in args.suites: diff --git a/scripts/automation/tests/__init__.py b/tools/automation/tests/__init__.py similarity index 100% rename from scripts/automation/tests/__init__.py rename to tools/automation/tests/__init__.py diff --git a/scripts/automation/tests/check_vcr_recordings.py b/tools/automation/tests/check_vcr_recordings.py similarity index 100% rename from scripts/automation/tests/check_vcr_recordings.py rename to tools/automation/tests/check_vcr_recordings.py diff --git a/scripts/automation/tests/nose_helper.py b/tools/automation/tests/nose_helper.py similarity index 100% rename from scripts/automation/tests/nose_helper.py rename to tools/automation/tests/nose_helper.py diff --git a/scripts/automation/tests/run.py b/tools/automation/tests/run.py similarity index 84% rename from scripts/automation/tests/run.py rename to tools/automation/tests/run.py index c2722a752af..1c3a929aa28 100644 --- a/scripts/automation/tests/run.py +++ b/tools/automation/tests/run.py @@ -80,16 +80,21 @@ def run_tests(modules, parallel, run_live, tests): help='The specific test to run in the given module. The string can represent a test class or a ' 'test class and a test method name. Multiple tests can be given, but they should all ' 'belong to one command modules.') + parse.add_argument('--ci', dest='ci', action='store_true', help='Run the tests in CI mode.') args = parse.parse_args() - if not args.modules and os.environ.get('AZURE_CLI_TEST_MODULES', None): - print('Test modules list is parsed from environment variable AZURE_CLI_TEST_MODULES.') - args.modules = [m.strip() for m in os.environ.get('AZURE_CLI_TEST_MODULES').split(',')] - - selected_modules = filter_user_selected_modules_with_tests(args.modules) - if not selected_modules: - parse.print_help() - sys.exit(1) + if args.ci: + print('Run tests in CI mode') + selected_modules = [('CI mode', 'azure.cli', 'azure.cli')] + else: + if not args.modules and os.environ.get('AZURE_CLI_TEST_MODULES', None): + print('Test modules list is parsed from environment variable AZURE_CLI_TEST_MODULES.') + args.modules = [m.strip() for m in os.environ.get('AZURE_CLI_TEST_MODULES').split(',')] + + selected_modules = filter_user_selected_modules_with_tests(args.modules) + if not selected_modules: + parse.print_help() + sys.exit(1) success = run_tests(selected_modules, parallel=args.parallel, run_live=args.live, tests=args.tests) diff --git a/scripts/automation/tests/verify_dependencies.py b/tools/automation/tests/verify_dependencies.py similarity index 100% rename from scripts/automation/tests/verify_dependencies.py rename to tools/automation/tests/verify_dependencies.py diff --git a/scripts/automation/tests/verify_packages.py b/tools/automation/tests/verify_packages.py similarity index 66% rename from scripts/automation/tests/verify_packages.py rename to tools/automation/tests/verify_packages.py index e7a1fdebdb6..76bf5cb6f3b 100644 --- a/scripts/automation/tests/verify_packages.py +++ b/tools/automation/tests/verify_packages.py @@ -31,6 +31,8 @@ Does setup.py include 'cmdclass=cmdclass'? """ +EXCLUDE_MODULES = set(['azure-cli-taskhelp']) + def exec_command(command, cwd=None, stdout=None, env=None): """Returns True in the command was executed successfully""" @@ -55,27 +57,6 @@ def set_version(path_to_setup): sys.stdout.write(line.replace('version=VERSION', "version='1000.0.0'")) -def build_package(path_to_package, dist_dir): - print_heading('Building {}'.format(path_to_package)) - path_to_setup = os.path.join(path_to_package, 'setup.py') - set_version(path_to_setup) - cmd_success = exec_command('python setup.py bdist_wheel -d {0}'.format(dist_dir), cwd=path_to_package) - if not cmd_success: - print_heading('Error building {}!'.format(path_to_package), f=sys.stderr) - sys.exit(1) - print_heading('Built {}'.format(path_to_package)) - - -def install_package(path_to_package, package_name, dist_dir): - print_heading('Installing {}'.format(path_to_package)) - cmd = 'python -m pip install --upgrade {} --find-links file://{}'.format(package_name, dist_dir) - cmd_success = exec_command(cmd) - if not cmd_success: - print_heading('Error installing {}!'.format(path_to_package), f=sys.stderr) - sys.exit(1) - print_heading('Installed {}'.format(path_to_package)) - - def _valid_wheel(wheel_path): # these files shouldn't exist in the wheel print('Verifying {}'.format(wheel_path)) @@ -88,20 +69,23 @@ def _valid_wheel(wheel_path): def run_help_on_command_without_err(command_str): - try: - subprocess.check_output(['az'] + command_str.split() + ['--help'], stderr=subprocess.STDOUT, - universal_newlines=True) - return True - except subprocess.CalledProcessError as err: - print(err.output, file=sys.stderr) - print(err, file=sys.stderr) - return False + for i in range(3): # retry the command 3 times + try: + command = 'az {} --help'.format(command_str).split() + subprocess.check_output(command, stderr=subprocess.STDOUT, universal_newlines=True) + + return True + except subprocess.CalledProcessError as error: + execution_error = error + continue + print(execution_error.output, file=sys.stderr) + print(execution_error, file=sys.stderr) -def verify_packages(): - # tmp dir to store all the built packages - built_packages_dir = tempfile.mkdtemp() + return False + +def verify_packages(built_packages_dir): all_modules = automation_path.get_all_module_paths() all_command_modules = automation_path.get_command_modules_paths(include_prefix=True) @@ -111,19 +95,7 @@ def verify_packages(): print(modules_missing_manifest_in) sys.exit(1) - # STEP 1:: Build the packages - for name, path in all_modules: - build_package(path, built_packages_dir) - - # STEP 2:: Install the CLI and dependencies - azure_cli_modules_path = next(path for name, path in all_modules if name == 'azure-cli') - install_package(azure_cli_modules_path, 'azure-cli', built_packages_dir) - - # Install the remaining command modules - for name, fullpath in all_command_modules: - install_package(fullpath, name, built_packages_dir) - - # STEP 3:: Validate the installation + # STEP 1:: Validate the installation try: az_output = subprocess.check_output(['az', '--debug'], stderr=subprocess.STDOUT, universal_newlines=True) @@ -137,23 +109,18 @@ def verify_packages(): print_heading('Error running the CLI!', f=sys.stderr) sys.exit(1) - # STEP 4:: Run -h on each command + # STEP 2:: Run -h on each command print('Running --help on all commands.') from azure.cli.core.application import Configuration config = Configuration() all_commands = list(config.get_command_table()) - pool_size = 5 - chunk_size = 10 command_results = [] - p = multiprocessing.Pool(pool_size) - prev_percent = 0 - for i, res in enumerate(p.imap_unordered(run_help_on_command_without_err, all_commands, chunk_size), 1): + p = multiprocessing.Pool(multiprocessing.cpu_count()) + for i, res in enumerate(p.imap_unordered(run_help_on_command_without_err, all_commands, 10), 1): + sys.stderr.write('{}/{}'.format(i, len(all_commands))) command_results.append(res) - cur_percent = int((i/len(all_commands))*100) - if cur_percent != prev_percent: - print('{}% complete'.format(cur_percent), file=sys.stderr) - prev_percent = cur_percent + p.close() p.join() if not all(command_results): @@ -161,7 +128,7 @@ def verify_packages(): sys.exit(1) print('OK.') - # STEP 5:: Determine if any modules failed to install + # STEP 3:: Determine if any modules failed to install pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources) installed_command_modules = [dist.key for dist in @@ -170,15 +137,15 @@ def verify_packages(): print('Installed command modules', installed_command_modules) - missing_modules = \ - set([name for name, fullpath in all_command_modules]) - set(installed_command_modules) + missing_modules = set([name for name, fullpath in all_command_modules]) - set(installed_command_modules) - \ + EXCLUDE_MODULES if missing_modules: print_heading('Error: The following modules were not installed successfully', f=sys.stderr) print(missing_modules, file=sys.stderr) sys.exit(1) - # STEP 6:: Verify the wheels that get produced + # STEP 4:: Verify the wheels that get produced print_heading('Verifying wheels...') invalid_wheels = [] for wheel_path in glob.glob(os.path.join(built_packages_dir, '*.whl')): @@ -196,4 +163,9 @@ def verify_packages(): if __name__ == '__main__': - verify_packages() + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('build_folder', help='The path to the folder contains all wheel files.') + + args = parser.parse_args() + verify_packages(args.build_folder) diff --git a/scripts/automation/tests/verify_readme_history.py b/tools/automation/tests/verify_readme_history.py similarity index 87% rename from scripts/automation/tests/verify_readme_history.py rename to tools/automation/tests/verify_readme_history.py index 61bfa7319e2..e07f7aef440 100644 --- a/scripts/automation/tests/verify_readme_history.py +++ b/tools/automation/tests/verify_readme_history.py @@ -7,7 +7,6 @@ from __future__ import print_function - import os import sys import argparse @@ -20,6 +19,7 @@ HISTORY_NAME = 'HISTORY.rst' RELEASE_HISTORY_TITLE = 'Release History' + def exec_command(command, cwd=None, stdout=None, env=None): """Returns True in the command was executed successfully""" try: @@ -42,15 +42,15 @@ def check_history_headings(mod_path): with open(history_path, 'r') as f: input_string = f.read() _, pub = core.publish_programmatically( - source_class=io.StringInput, source=input_string, - source_path=source_path, - destination_class=io.NullOutput, destination=None, - destination_path=destination_path, - reader=None, reader_name='standalone', - parser=None, parser_name='restructuredtext', - writer=None, writer_name='null', - settings=None, settings_spec=None, settings_overrides={}, - config_section=None, enable_exit_status=None) + source_class=io.StringInput, source=input_string, + source_path=source_path, + destination_class=io.NullOutput, destination=None, + destination_path=destination_path, + reader=None, reader_name='standalone', + parser=None, parser_name='restructuredtext', + writer=None, writer_name='null', + settings=None, settings_spec=None, settings_overrides={}, + config_section=None, enable_exit_status=None) if pub.writer.document.children[0].rawsource == RELEASE_HISTORY_TITLE: return True else: @@ -64,6 +64,7 @@ def check_readme_render(mod_path): checks.append(check_history_headings(mod_path)) return all(checks) + def verify_all(): all_paths = get_all_module_paths() all_ok = [] @@ -84,6 +85,7 @@ def verify_all(): else: print('Verified READMEs of all modules successfully.', file=sys.stderr) + def verify_one(mod_name): p = [path for name, path in get_all_module_paths() if name == mod_name] assert p, 'Module not found.' @@ -94,6 +96,7 @@ def verify_one(mod_name): print('note: Line numbers in the errors map to the long_description of your setup.py.') sys.exit(1) + if __name__ == '__main__': parser = argparse.ArgumentParser( description="Verify the README and HISTORY files for each module so they format correctly on PyPI.") diff --git a/scripts/automation/utilities/__init__.py b/tools/automation/utilities/__init__.py similarity index 100% rename from scripts/automation/utilities/__init__.py rename to tools/automation/utilities/__init__.py diff --git a/scripts/automation/utilities/const.py b/tools/automation/utilities/const.py similarity index 100% rename from scripts/automation/utilities/const.py rename to tools/automation/utilities/const.py diff --git a/scripts/automation/utilities/display.py b/tools/automation/utilities/display.py similarity index 100% rename from scripts/automation/utilities/display.py rename to tools/automation/utilities/display.py diff --git a/scripts/automation/utilities/path.py b/tools/automation/utilities/path.py similarity index 100% rename from scripts/automation/utilities/path.py rename to tools/automation/utilities/path.py diff --git a/tools/automation/verify/__init__.py b/tools/automation/verify/__init__.py new file mode 100644 index 00000000000..28291fdd92f --- /dev/null +++ b/tools/automation/verify/__init__.py @@ -0,0 +1,57 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + + +from automation.verify.doc_source_map import verify_doc_source_map +from automation.verify.default_modules import verify_default_modules + + +def verify_license(args): + import sys + import os + from automation.utilities.path import get_repo_root + + license_header = \ +"""# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +""" + + env_path = os.path.join(get_repo_root(), 'env') + + files_without_header = [] + for current_dir, _, files in os.walk(get_repo_root()): + if current_dir.startswith(env_path): + continue + + file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py') and p != 'azure_bdist_wheel.py') + for python_file in file_itr: + with open(python_file, 'r') as f: + file_text = f.read() + + if file_text and license_header not in file_text: + files_without_header.append(os.path.join(current_dir, python_file)) + + if files_without_header: + sys.stderr.write("Error: The following files don't have the required license headers: \n{}".format( + '\n'.join(files_without_header))) + + sys.exit(1) + + +def init_args(root): + parser = root.add_parser('verify') + sub_parser = parser.add_subparsers() + + license_verify = sub_parser.add_parser('license', help='Verify license headers.') + license_verify.set_defaults(func=verify_license) + + doc_map = sub_parser.add_parser('document-map', help='Verify documentation map.') + doc_map.set_defaults(func=verify_doc_source_map) + + def_modules = sub_parser.add_parser('default-modules', help='Verify default modules.') + def_modules.add_argument('build_folder', help='The path to the folder contains all wheel files.') + def_modules.set_defaults(func=verify_default_modules) diff --git a/scripts/automation/tests/verify_default_modules.py b/tools/automation/verify/default_modules.py similarity index 60% rename from scripts/automation/tests/verify_default_modules.py rename to tools/automation/verify/default_modules.py index e864d9dfe51..f405599cacf 100644 --- a/scripts/automation/tests/verify_default_modules.py +++ b/tools/automation/verify/default_modules.py @@ -12,39 +12,39 @@ import json import tempfile import zipfile -import subprocess +import glob -from ..utilities.path import (get_repo_root, get_command_modules_paths) -from ..utilities.display import print_heading -from ..utilities.const import COMMAND_MODULE_PREFIX +from automation.utilities.path import (get_repo_root, get_command_modules_paths) +from automation.utilities.display import print_heading -REPO_ROOT = get_repo_root() -AZURE_CLI_PATH = os.path.join(REPO_ROOT, 'src', 'azure-cli') +AZURE_CLI_PATH = os.path.join(get_repo_root(), 'src', 'azure-cli') AZURE_CLI_SETUP_PY = os.path.join(AZURE_CLI_PATH, 'setup.py') # This is a list of modules that we do not want to be installed by default. # Add your modules to this list if you don't want it to be installed when the CLI is installed. MODULES_TO_EXCLUDE = ['azure-cli-taskhelp'] -def get_cli_dependencies(): - dist_dir = tempfile.mkdtemp() + +def get_cli_dependencies(build_folder): + azure_cli_wheel = glob.glob(build_folder.rstrip('/') + '/azure_cli-*.whl')[0] + print('Explore wheel file {}.'.format(azure_cli_wheel)) + tmp_dir = tempfile.mkdtemp() - try: - subprocess.check_call(['python', 'setup.py', 'bdist_wheel', '-d', dist_dir], cwd=AZURE_CLI_PATH) - wheel_path = os.path.join(dist_dir, os.listdir(dist_dir)[0]) - zip_ref = zipfile.ZipFile(wheel_path, 'r') - zip_ref.extractall(tmp_dir) - zip_ref.close() - dist_info_dir = [f for f in os.listdir(tmp_dir) if f.endswith('.dist-info')][0] - whl_metadata_filepath = os.path.join(tmp_dir, dist_info_dir, 'metadata.json') - with open(whl_metadata_filepath) as f: - return json.load(f)['run_requires'][0]['requires'] - except subprocess.CalledProcessError as err: - print(err, file=sys.stderr) - -if __name__ == '__main__': + + zip_ref = zipfile.ZipFile(azure_cli_wheel, 'r') + zip_ref.extractall(tmp_dir) + zip_ref.close() + + dist_info_dir = [f for f in os.listdir(tmp_dir) if f.endswith('.dist-info')][0] + whl_metadata_filepath = os.path.join(tmp_dir, dist_info_dir, 'metadata.json') + with open(whl_metadata_filepath) as f: + print('Load metadata file from {}'.format(whl_metadata_filepath)) + return json.load(f)['run_requires'][0]['requires'] + + +def verify_default_modules(args): errors_list = [] - cli_deps = get_cli_dependencies() + cli_deps = get_cli_dependencies(args.build_folder) all_command_modules = get_command_modules_paths(include_prefix=True) if not cli_deps: print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_SETUP_PY), file=sys.stderr) @@ -54,7 +54,9 @@ def get_cli_dependencies(): errors_list.append("{} is a dependency of azure-cli BUT is marked as should be excluded.".format(modname)) if modname not in cli_deps and modname not in MODULES_TO_EXCLUDE: errors_list.append("{} is not included to be installed by default! If this is a mistake, modify {}. " - "Otherwise, modify this script ({}) to exclude the module.".format(modname, AZURE_CLI_SETUP_PY, __file__)) + "Otherwise, modify this script ({}) to exclude the module.".format(modname, + AZURE_CLI_SETUP_PY, + __file__)) if errors_list: print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_SETUP_PY)) print('\n'.join(errors_list), file=sys.stderr) diff --git a/scripts/automation/tests/verify_doc_source_map.py b/tools/automation/verify/doc_source_map.py similarity index 89% rename from scripts/automation/tests/verify_doc_source_map.py rename to tools/automation/verify/doc_source_map.py index 763bf3afb8f..0053aee1307 100644 --- a/scripts/automation/tests/verify_doc_source_map.py +++ b/tools/automation/verify/doc_source_map.py @@ -20,11 +20,13 @@ HELP_FILE_NAME = '_help.py' DOC_SOURCE_MAP_PATH = os.path.join('doc', 'sphinx', 'azhelpgen', DOC_MAP_NAME) + def _get_help_files_in_map(map_path): with open(map_path) as json_file: json_data = json.load(json_file) return list(json_data.values()) + def _map_help_files_not_found(help_files_in_map): none_existent_files = [] for f in help_files_in_map: @@ -32,6 +34,7 @@ def _map_help_files_not_found(help_files_in_map): none_existent_files.append(f) return none_existent_files + def _help_files_not_in_map(help_files_in_map): found_files = [] not_in_map = [] @@ -46,15 +49,13 @@ def _help_files_not_in_map(help_files_in_map): not_in_map.append(f_path) return not_in_map -def verify_doc_source_map(): + +def verify_doc_source_map(*args): map_path = os.path.join(REPO_ROOT, DOC_SOURCE_MAP_PATH) help_files_in_map = _get_help_files_in_map(map_path) - none_existent_files = _map_help_files_not_found(help_files_in_map) - not_in_map = _help_files_not_in_map(help_files_in_map) - return none_existent_files, not_in_map + help_files_not_found = _map_help_files_not_found(help_files_in_map) + hep_files_to_add_to_map = _help_files_not_in_map(help_files_in_map) -if __name__ == '__main__': - help_files_not_found, hep_files_to_add_to_map = verify_doc_source_map() if help_files_not_found or hep_files_to_add_to_map: print_heading('Errors whilst verifying {}!'.format(DOC_MAP_NAME)) if help_files_not_found: @@ -66,4 +67,3 @@ def verify_doc_source_map(): sys.exit(1) else: print('Verified {} successfully.'.format(DOC_MAP_NAME), file=sys.stderr) - diff --git a/tools/scripts/azdev b/tools/scripts/azdev new file mode 100755 index 00000000000..a2781e004fc --- /dev/null +++ b/tools/scripts/azdev @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +python -m automation "$@" diff --git a/tools/scripts/azdev.bat b/tools/scripts/azdev.bat new file mode 100644 index 00000000000..1b9cccdaf95 --- /dev/null +++ b/tools/scripts/azdev.bat @@ -0,0 +1,9 @@ +@echo off + +REM -------------------------------------------------------------------------------------------- +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. See License.txt in the project root for license information. +REM -------------------------------------------------------------------------------------------- + +pip show azure-cli-dev-tools>NUL || pip install -e %~dp0 +python -m automation %* \ No newline at end of file diff --git a/scripts/check_style b/tools/scripts/check_style similarity index 100% rename from scripts/check_style rename to tools/scripts/check_style diff --git a/scripts/check_style.bat b/tools/scripts/check_style.bat similarity index 100% rename from scripts/check_style.bat rename to tools/scripts/check_style.bat diff --git a/scripts/run_tests b/tools/scripts/run_tests similarity index 100% rename from scripts/run_tests rename to tools/scripts/run_tests diff --git a/scripts/run_tests.bat b/tools/scripts/run_tests.bat similarity index 85% rename from scripts/run_tests.bat rename to tools/scripts/run_tests.bat index ac845949c25..d4d8698d4ab 100644 --- a/scripts/run_tests.bat +++ b/tools/scripts/run_tests.bat @@ -5,5 +5,5 @@ REM Copyright (c) Microsoft Corporation. All rights reserved. REM Licensed under the MIT License. See License.txt in the project root for license information. REM -------------------------------------------------------------------------------------------- -pip show azure-cli-utility-automation>NUL || pip install -e %~dp0 +pip show azure-cli-dev-tools>NUL || pip install -e %~dp0 python -m automation.tests.run %* \ No newline at end of file diff --git a/scripts/setup.py b/tools/setup.py similarity index 83% rename from scripts/setup.py rename to tools/setup.py index 04e3bd5c964..0bbcdbdc188 100644 --- a/scripts/setup.py +++ b/tools/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. @@ -25,6 +23,7 @@ DEPENDENCIES = [ 'autopep8>=1.2.4', + 'pylint>=1.7.1' 'coverage>=4.2', 'flake8>=3.2.1', 'pycodestyle>=2.2.0', @@ -35,9 +34,9 @@ ] setup( - name='azure-cli-utility-automation', + name='azure-cli-dev-tools', version=VERSION, - description='Microsoft Azure Command-Line Tools - Automation Utility', + description='Microsoft Azure Command-Line - Development Tools', long_description='', license='MIT', author='Microsoft Corporation', @@ -46,10 +45,12 @@ namespace_packages=[ ], scripts=[ - "check_style", - "check_style.bat", - "run_tests", - "run_tests.bat" + "scripts/check_style", + "scripts/check_style.bat", + "scripts/run_tests", + "scripts/run_tests.bat", + 'scripts/azdev', + 'scripts/azdev.bat' ], packages=[ 'automation',