forked from Azure/azure-cli
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
1. Adopt build stage feature. Separate the build process into build, verify and publish stages. 2. Move build utilties from scripts folder to its own tools folder for better organization. 3. Publish wheel artifacts to a blob storage container for long-term persistence.
- Loading branch information
Showing
61 changed files
with
496 additions
and
234 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,27 +1,37 @@ | ||
sudo: false | ||
dist: trusty | ||
sudo: off | ||
language: python | ||
addons: | ||
apt: | ||
packages: | ||
- libssl-dev | ||
- libffi-dev | ||
- python-dev | ||
install: echo 'skip' | ||
cache: pip | ||
jobs: | ||
include: | ||
- stage: all | ||
env: CODE_COVERAGE="True" | ||
python: 2.7 | ||
install: python scripts/dev_setup.py | ||
script: scripts/build.sh | ||
- | ||
env: CODE_COVERAGE="True" | ||
python: 3.5 | ||
install: python scripts/dev_setup.py | ||
script: scripts/build.sh | ||
- | ||
env: CODE_COVERAGE="True" | ||
python: 3.6 | ||
install: python scripts/dev_setup.py | ||
script: scripts/build.sh | ||
- | ||
python: 2.7 | ||
install: pip install -qqq virtualenv | ||
script: scripts/package_verify.sh | ||
- | ||
python: 3.6 | ||
install: pip install -qqq virtualenv | ||
script: scripts/package_verify.sh | ||
- stage: build | ||
script: ./scripts/ci/build.sh | ||
- stage: verify | ||
env: PURPOSE='Automation' | ||
script: ./scripts/ci/test_automation.sh | ||
python: 3.6 | ||
- stage: verify | ||
env: PURPOSE='Automation' | ||
script: ./scripts/ci/test_automation.sh | ||
python: 2.7 | ||
- stage: verify | ||
script: ./scripts/ci/test_static.sh | ||
env: PURPOSE='Static Check' | ||
python: 3.6 | ||
- stage: verify | ||
script: ./scripts/ci/test_integration.sh | ||
env: PURPOSE='Integration' | ||
python: 3.6 | ||
- stage: verify | ||
script: ./scripts/ci/test_integration.sh | ||
env: PURPOSE='Integration' | ||
python: 2.7 | ||
- stage: publish | ||
script: ./scripts/ci/publish.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
# Build wheel and set the $share_folder to the artifacts folder | ||
|
||
. $(cd $(dirname $0); pwd)/build.sh | ||
share_folder=$(cd $(dirname $0); cd ../../artifacts; pwd) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Build wheel packages containing both CLI product and tests. The script doesn't rely on a pre-existing virtual | ||
# environment. | ||
|
||
set -e | ||
|
||
############################################## | ||
# clean up | ||
mkdir -p ./artifacts | ||
echo `git rev-parse --verify HEAD` > ./artifacts/build.sha | ||
|
||
mkdir -p ./artifacts/build | ||
mkdir -p ./artifacts/app | ||
mkdir -p ./artifacts/testsrc | ||
|
||
output_dir=$(cd artifacts/build && pwd) | ||
testsrc_dir=$(cd artifacts/testsrc && pwd) | ||
app_dir=$(cd artifacts/app && pwd) | ||
|
||
############################################## | ||
# Copy app scripts for batch run - To be retired in the future | ||
cp $(cd $(dirname $0); pwd)/app/* $app_dir | ||
|
||
############################################## | ||
# List tests to the test file - To be replaced by other mechanism in the fucture | ||
# content=`nosetests azure.cli -v --collect-only 2>&1` | ||
|
||
# echo $content > $app_dir/raw.txt | ||
# manifest="$app_dir/all_tests.txt" | ||
# if [ -e $manifest ]; then rm $manifest; fi | ||
|
||
# while IFS='' read -r line || [[ -n "$line" ]]; do | ||
# if [ -n "$line" ] && [ "OK" != "$line" ] && [ "${line:0:3}" != "Ran" ] && [ "${line:0:3}" != "---" ]; then | ||
# parts=($line) | ||
# test_method=${parts[0]} | ||
# test_class=${parts[1]} | ||
# test_class=${test_class##(} | ||
# test_class=${test_class%%)} | ||
|
||
# echo "$test_class $test_method" >> $manifest | ||
# fi | ||
# done <<< "$content" | ||
|
||
############################################## | ||
# build product packages | ||
echo 'Build Azure CLI and its command modules' | ||
for setup_file in $(find src -name 'setup.py'); do | ||
pushd $(dirname $setup_file) | ||
echo "" | ||
echo "Building module at $(pwd) ..." | ||
python setup.py bdist_wheel -d $output_dir sdist -d $output_dir | ||
popd | ||
done | ||
|
||
############################################## | ||
# build test packages | ||
echo 'Build Azure CLI tests package' | ||
|
||
echo "Copy test source code into $build_src ..." | ||
for test_src in $(find src/command_modules -name tests -type d); do | ||
rel_path=${test_src##src/command_modules/} | ||
rel_path=(${rel_path/\// }) | ||
rel_path=${rel_path[1]} | ||
|
||
mkdir -p $testsrc_dir/$rel_path | ||
cp -R $test_src/* $testsrc_dir/$rel_path | ||
done | ||
|
||
for test_src in $(find src -name tests | grep -v command_modules); do | ||
rel_path=${test_src##src/} | ||
rel_path=(${rel_path/\// }) | ||
rel_path=${rel_path[1]} | ||
|
||
mkdir -p $testsrc_dir/$rel_path | ||
cp -R $test_src/* $testsrc_dir/$rel_path | ||
done | ||
|
||
cat >$testsrc_dir/setup.py <<EOL | ||
#!/usr/bin/env python | ||
from setuptools import setup | ||
VERSION = "1.0.0+dev" | ||
CLASSIFIERS = [ | ||
'Development Status :: 3 - Alpha', | ||
'Intended Audience :: Developers', | ||
'Programming Language :: Python', | ||
'Programming Language :: Python :: 2', | ||
'Programming Language :: Python :: 2.7', | ||
'Programming Language :: Python :: 3', | ||
'Programming Language :: Python :: 3.4', | ||
'Programming Language :: Python :: 3.5', | ||
'Programming Language :: Python :: 3.6', | ||
'License :: OSI Approved :: MIT License', | ||
] | ||
DEPENDENCIES = [ | ||
'azure-cli', | ||
'azure-cli-testsdk' | ||
] | ||
setup( | ||
name='azure-cli-fulltest', | ||
version=VERSION, | ||
description='Microsoft Azure Command-Line Tools Full Tests', | ||
license='MIT', | ||
author='Microsoft Corporation', | ||
author_email='azpycli@microsoft.com', | ||
url='https://github.com/Azure/azure-cli', | ||
zip_safe=False, | ||
classifiers=CLASSIFIERS, | ||
packages=[ | ||
'azure.cli.core.tests', | ||
EOL | ||
|
||
for name in $(ls src/command_modules | grep azure-cli-); do | ||
module_name=${name##azure-cli-} | ||
echo "src/command_modules/azure-cli-$name/azure/cli/command_modules/$name/tests" | ||
if [ -d src/command_modules/$name/azure/cli/command_modules/$module_name/tests ]; then | ||
echo " 'azure.cli.command_modules.$module_name.tests'," >>$testsrc_dir/setup.py | ||
fi | ||
done | ||
|
||
|
||
cat >>$testsrc_dir/setup.py <<EOL | ||
], | ||
package_data={'': ['recordings/**/*.yaml', | ||
'data/*.zip', | ||
'data/*.whl', | ||
'*.pem', | ||
'*.pfx', | ||
'*.txt', | ||
'*.json', | ||
'*.byok', | ||
'*.js', | ||
'*.md', | ||
'*.bat', | ||
'*.txt', | ||
'*.cer', | ||
'**/*.cer', | ||
'**/*.pem', | ||
'**/*.pfx', | ||
'**/*.txt', | ||
'**/*.json', | ||
'**/*.byok', | ||
'**/*.js', | ||
'**/*.md', | ||
'**/*.bat', | ||
'**/*.txt']}, | ||
install_requires=DEPENDENCIES | ||
) | ||
EOL | ||
|
||
pushd $testsrc_dir | ||
python setup.py bdist_wheel -d $output_dir sdist -d $output_dir | ||
popd | ||
|
||
rm -rf $testsrc_dir # clear afterwards |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -e | ||
|
||
unset AZURE_CLI_DIAGNOSTICS_TELEMETRY | ||
|
||
if [ -z $PUBLISH_STORAGE_SAS ] || [ -z $PUBLISH_STORAGE_ACCT ] || [ -z $PUBLISH_CONTAINER ]; then | ||
echo 'Missing publish storage account credential. Skip publishing.' | ||
exit 0 | ||
fi | ||
|
||
echo 'Generate artifacts' | ||
. $(cd $(dirname $0); pwd)/artifacts.sh | ||
|
||
echo 'Set up output folder' | ||
mkdir -p ./publish/$TRAVIS_REPO_SLUG/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER | ||
cp -R $share_folder/* ./publish/$TRAVIS_REPO_SLUG/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER | ||
|
||
echo 'Listing output folder' | ||
ls -R ./publish | ||
|
||
echo 'Installing public Azure CLI for uploading operation.' | ||
pip install -qqq azure-cli | ||
|
||
echo 'Uploading ...' | ||
az storage blob upload-batch -s ./publish \ | ||
-d $PUBLISH_CONTAINER \ | ||
--account-name $PUBLISH_STORAGE_ACCT \ | ||
--sas-token $PUBLISH_STORAGE_SAS \ | ||
-otable |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -e | ||
|
||
. $(cd $(dirname $0); pwd)/artifacts.sh | ||
|
||
pip install -qqq -e ./tools | ||
pip install -qqq coverage codecov | ||
pip install -qqq azure-cli-fulltest -f $share_folder/build | ||
|
||
echo '=== List installed packages' | ||
pip freeze | ||
|
||
echo '=== Begin testing' | ||
coverage run -m automation.tests.run --parallel --ci | ||
coverage combine | ||
codecov |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -e | ||
|
||
. $(cd $(dirname $0); pwd)/artifacts.sh | ||
|
||
pip install -e ./tools | ||
pip install azure-cli -f $share_folder/build | ||
|
||
echo '=== List installed packages' | ||
pip freeze | ||
|
||
echo '=== Begin testing' | ||
python -m automation.tests.verify_packages $share_folder/build | ||
python -m automation.tests.verify_dependencies |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -e | ||
|
||
. $(cd $(dirname $0); pwd)/artifacts.sh | ||
|
||
pip install pylint flake8 | ||
pip install azure-cli-fulltest -f $share_folder/build | ||
|
||
echo '=== List installed packages' | ||
pip freeze | ||
|
||
echo '=== Begin testing' | ||
|
||
proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'` | ||
|
||
echo "Run pylint with $proc_number proc." | ||
pylint azure.cli --rcfile=./pylintrc -j $proc_number | ||
|
||
echo "Run flake8." | ||
flake8 --statistics --exclude=azure_bdist_wheel.py --append-config=./.flake8 ./src | ||
|
||
pip install -e ./tools | ||
echo "Scan license" | ||
azdev verify license | ||
|
||
echo "Documentation Map" | ||
azdev verify document-map | ||
|
||
echo "Command lint" | ||
python -m automation.commandlint.run | ||
|
||
echo "Verify readme history" | ||
python -m automation.tests.verify_readme_history | ||
|
||
echo "Verify default modules" | ||
azdev verify default-modules $share_folder/build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.