Overhaul Travis build configuration

1. Adopt build stage feature. Separate the build process into build,
verify and publish stages.
2. Move build utilties from scripts folder to its own tools folder for
better organization.
3. Publish wheel artifacts to a blob storage container for long-term
persistence.

.travis.yml

@@ -1,27 +1,37 @@
-sudo: false
+dist: trusty
+sudo: off
 language: python
+addons:
+  apt:
+    packages:
+      - libssl-dev
+      - libffi-dev
+      - python-dev
+install: echo 'skip'
+cache: pip
 jobs:
   include:
-   - stage: all
-     env: CODE_COVERAGE="True"
-     python: 2.7
-     install: python scripts/dev_setup.py
-     script: scripts/build.sh
-   -
-     env: CODE_COVERAGE="True"
-     python: 3.5
-     install: python scripts/dev_setup.py
-     script: scripts/build.sh
-   -
-     env: CODE_COVERAGE="True"
-     python: 3.6
-     install: python scripts/dev_setup.py
-     script: scripts/build.sh
-   -
-     python: 2.7
-     install: pip install -qqq virtualenv
-     script: scripts/package_verify.sh
-   -
-     python: 3.6
-     install: pip install -qqq virtualenv
-     script: scripts/package_verify.sh
+    - stage: build
+      script: ./scripts/ci/build.sh
+    - stage: verify
+      env: PURPOSE='Automation'
+      script: ./scripts/ci/test_automation.sh
+      python: 3.6
+    - stage: verify
+      env: PURPOSE='Automation'
+      script: ./scripts/ci/test_automation.sh
+      python: 2.7
+    - stage: verify
+      script: ./scripts/ci/test_static.sh
+      env: PURPOSE='Static Check'
+      python: 3.6
+    - stage: verify
+      script: ./scripts/ci/test_integration.sh
+      env: PURPOSE='Integration'
+      python: 3.6
+    - stage: verify
+      script: ./scripts/ci/test_integration.sh
+      env: PURPOSE='Integration'
+      python: 2.7
+    - stage: publish
+      script: ./scripts/ci/publish.sh

pylintrc

@@ -7,7 +7,7 @@ reports=no
 # locally-disabled: Warning locally suppressed using disable-msg
 # cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
 # too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
-disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name
+disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code
 
 [FORMAT]
 max-line-length=120

requirements.txt

@@ -8,7 +8,6 @@ mock==1.3.0
 paramiko==2.0.2
 pip==9.0.1
 pygments==2.1.3
-pylint==1.7.1
 pyOpenSSL==16.2.0
 pyyaml==3.11
 requests==2.9.1

scripts/ci/artifacts.sh

@@ -0,0 +1,4 @@
+# Build wheel and set the $share_folder to the artifacts folder
+
+. $(cd $(dirname $0); pwd)/build.sh
+share_folder=$(cd $(dirname $0); cd ../../artifacts; pwd)

scripts/ci/build.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+# Build wheel packages containing both CLI product and tests. The script doesn't rely on a pre-existing virtual
+# environment.
+
+set -e
+
+##############################################
+# clean up
+mkdir -p ./artifacts
+echo `git rev-parse --verify HEAD` > ./artifacts/build.sha
+
+mkdir -p ./artifacts/build
+mkdir -p ./artifacts/app
+mkdir -p ./artifacts/testsrc
+
+output_dir=$(cd artifacts/build && pwd)
+testsrc_dir=$(cd artifacts/testsrc && pwd)
+app_dir=$(cd artifacts/app && pwd)
+
+##############################################
+# Copy app scripts for batch run - To be retired in the future
+cp $(cd $(dirname $0); pwd)/app/* $app_dir
+
+##############################################
+# List tests to the test file - To be replaced by other mechanism in the fucture
+# content=`nosetests azure.cli -v --collect-only 2>&1`
+
+# echo $content > $app_dir/raw.txt
+# manifest="$app_dir/all_tests.txt"
+# if [ -e $manifest ]; then rm $manifest; fi
+
+# while IFS='' read -r line || [[ -n "$line" ]]; do
+#     if [ -n "$line" ] && [ "OK" != "$line" ] && [ "${line:0:3}" != "Ran" ] && [ "${line:0:3}" != "---" ]; then
+#         parts=($line)
+#         test_method=${parts[0]}
+#         test_class=${parts[1]}
+#         test_class=${test_class##(}
+#         test_class=${test_class%%)}
+
+#         echo "$test_class $test_method" >> $manifest
+#     fi
+# done <<< "$content"
+
+##############################################
+# build product packages
+echo 'Build Azure CLI and its command modules'
+for setup_file in $(find src -name 'setup.py'); do
+    pushd $(dirname $setup_file)
+    echo ""
+    echo "Building module at $(pwd) ..."
+    python setup.py bdist_wheel -d $output_dir sdist -d $output_dir
+    popd
+done
+
+##############################################
+# build test packages
+echo 'Build Azure CLI tests package'
+
+echo "Copy test source code into $build_src ..."
+for test_src in $(find src/command_modules -name tests -type d); do
+    rel_path=${test_src##src/command_modules/}
+    rel_path=(${rel_path/\// })
+    rel_path=${rel_path[1]}
+
+    mkdir -p $testsrc_dir/$rel_path
+    cp -R $test_src/* $testsrc_dir/$rel_path
+done
+
+for test_src in $(find src -name tests | grep -v command_modules); do
+    rel_path=${test_src##src/}
+    rel_path=(${rel_path/\// })
+    rel_path=${rel_path[1]}
+
+    mkdir -p $testsrc_dir/$rel_path
+    cp -R $test_src/* $testsrc_dir/$rel_path
+done
+
+cat >$testsrc_dir/setup.py <<EOL
+#!/usr/bin/env python
+
+from setuptools import setup
+
+VERSION = "1.0.0+dev"
+
+CLASSIFIERS = [
+    'Development Status :: 3 - Alpha',
+    'Intended Audience :: Developers',
+    'Programming Language :: Python',
+    'Programming Language :: Python :: 2',
+    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.4',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'License :: OSI Approved :: MIT License',
+]
+
+DEPENDENCIES = [
+    'azure-cli',
+    'azure-cli-testsdk'
+]
+
+setup(
+    name='azure-cli-fulltest',
+    version=VERSION,
+    description='Microsoft Azure Command-Line Tools Full Tests',
+    license='MIT',
+    author='Microsoft Corporation',
+    author_email='azpycli@microsoft.com',
+    url='https://github.com/Azure/azure-cli',
+    zip_safe=False,
+    classifiers=CLASSIFIERS,
+    packages=[
+        'azure.cli.core.tests',
+EOL
+
+for name in $(ls src/command_modules | grep azure-cli-); do
+    module_name=${name##azure-cli-}
+    echo "src/command_modules/azure-cli-$name/azure/cli/command_modules/$name/tests"
+    if [ -d src/command_modules/$name/azure/cli/command_modules/$module_name/tests ]; then
+        echo "        'azure.cli.command_modules.$module_name.tests'," >>$testsrc_dir/setup.py
+    fi
+done
+
+
+cat >>$testsrc_dir/setup.py <<EOL
+    ],
+    package_data={'': ['recordings/**/*.yaml', 
+                       'data/*.zip',
+                       'data/*.whl',
+                       '*.pem', 
+                       '*.pfx',
+                       '*.txt',
+                       '*.json',
+                       '*.byok',
+                       '*.js',
+                       '*.md',
+                       '*.bat',
+                       '*.txt',
+                       '*.cer', 
+                       '**/*.cer',
+                       '**/*.pem', 
+                       '**/*.pfx',
+                       '**/*.txt',
+                       '**/*.json',
+                       '**/*.byok',
+                       '**/*.js',
+                       '**/*.md',
+                       '**/*.bat',
+                       '**/*.txt']},
+    install_requires=DEPENDENCIES
+)
+EOL
+
+pushd $testsrc_dir
+python setup.py bdist_wheel -d $output_dir sdist -d $output_dir
+popd
+
+rm -rf $testsrc_dir  # clear afterwards

scripts/ci/publish.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -e
+
+unset AZURE_CLI_DIAGNOSTICS_TELEMETRY
+
+if [ -z $PUBLISH_STORAGE_SAS ] || [ -z $PUBLISH_STORAGE_ACCT ] || [ -z $PUBLISH_CONTAINER ]; then
+    echo 'Missing publish storage account credential. Skip publishing.'
+    exit 0
+fi
+
+echo 'Generate artifacts'
+. $(cd $(dirname $0); pwd)/artifacts.sh 
+
+echo 'Set up output folder'
+mkdir -p ./publish/$TRAVIS_REPO_SLUG/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER
+cp -R $share_folder/* ./publish/$TRAVIS_REPO_SLUG/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER
+
+echo 'Listing output folder'
+ls -R ./publish
+
+echo 'Installing public Azure CLI for uploading operation.'
+pip install -qqq azure-cli
+
+echo 'Uploading ...'
+az storage blob upload-batch -s ./publish \
+                             -d $PUBLISH_CONTAINER \
+                             --account-name $PUBLISH_STORAGE_ACCT \
+                             --sas-token $PUBLISH_STORAGE_SAS \
+                             -otable

scripts/ci/test_automation.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+. $(cd $(dirname $0); pwd)/artifacts.sh
+
+pip install -qqq -e ./tools
+pip install -qqq coverage codecov
+pip install -qqq azure-cli-fulltest -f $share_folder/build
+
+echo '=== List installed packages'
+pip freeze
+
+echo '=== Begin testing'
+coverage run -m automation.tests.run --parallel --ci
+coverage combine
+codecov

scripts/ci/test_integration.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+. $(cd $(dirname $0); pwd)/artifacts.sh
+
+pip install -e ./tools
+pip install azure-cli -f $share_folder/build
+
+echo '=== List installed packages'
+pip freeze
+
+echo '=== Begin testing'
+python -m automation.tests.verify_packages $share_folder/build
+python -m automation.tests.verify_dependencies

scripts/ci/test_static.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+. $(cd $(dirname $0); pwd)/artifacts.sh
+
+pip install pylint flake8
+pip install azure-cli-fulltest -f $share_folder/build
+
+echo '=== List installed packages'
+pip freeze
+
+echo '=== Begin testing'
+
+proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'`
+
+echo "Run pylint with $proc_number proc."
+pylint azure.cli --rcfile=./pylintrc -j $proc_number
+
+echo "Run flake8."
+flake8 --statistics --exclude=azure_bdist_wheel.py --append-config=./.flake8 ./src
+
+pip install -e ./tools
+echo "Scan license"
+azdev verify license
+
+echo "Documentation Map"
+azdev verify document-map
+
+echo "Command lint"
+python -m automation.commandlint.run 
+
+echo "Verify readme history"
+python -m automation.tests.verify_readme_history
+
+echo "Verify default modules"
+azdev verify default-modules $share_folder/build

scripts/dev_setup.py

@@ -30,7 +30,7 @@ def exec_command(command):
 exec_command('pip install -r requirements.txt')
 
 # install automation package
-exec_command('pip install -e ./scripts')
+exec_command('pip install -e ./tools')
 
 # command modules have dependency on azure-cli-core so install this first
 exec_command('pip install -e src/azure-cli-nspkg')