From fa998ce7dcd341cd6b1f1876d6f77284a382261a Mon Sep 17 00:00:00 2001 From: FumingZhang <81607949+FumingZhang@users.noreply.github.com> Date: Wed, 1 Dec 2021 14:48:29 +0800 Subject: [PATCH] {AKS} Update live test pipeline (#4171) * update pipeline * Trigger Build * fix sh * Trigger Build --- .../scripts/test_cli_live.sh | 93 ++++++++-- .../scripts/test_cli_unit.sh | 11 +- .../scripts/test_ext_live.sh | 164 +++++++++++++++--- .../scripts/test_ext_unit.sh | 11 +- .../scripts/test_raw_cases.sh | 7 - .../scripts/transcribe_env.sh | 80 ++++++--- 6 files changed, 281 insertions(+), 85 deletions(-) delete mode 100644 src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh index d7d2b30e9f5..a81efc06a4e 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh @@ -7,15 +7,27 @@ set -o pipefail set -o xtrace # check var +[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) [[ -z "${TENANT_ID}" ]] && (echo "TENANT_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_SUBSCRIPTION_ID}" ]] && (echo "AZCLI_ALT_SUBSCRIPTION_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_CLIENT_ID}" ]] && (echo "AZCLI_ALT_CLIENT_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "AZCLI_ALT_CLIENT_SECRET is empty"; exit 1) +[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty") [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1) -[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1) +[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty") +[[ -z "${RERUNS}" ]] && (echo "RERUNS is empty") +[[ -z "${NON_BOXED}" ]] && (echo "NON_BOXED is empty") +[[ -z "${LOG_LEVEL}" ]] && (echo "LOG_LEVEL is empty") +[[ -z "${CAPTURE}" ]] && (echo "CAPTURE is empty") +[[ -z "${NO_EXIT_FIRST}" ]] && (echo "NO_EXIT_FIRST is empty") +[[ -z "${LAST_FAILED}" ]] && (echo "LAST_FAILED is empty") [[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty") [[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty") [[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty") +[[ -z "${ENABLE_SELECTION}" ]] && (echo "ENABLE_SELECTION is empty") +[[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty") +[[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty") +[[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty") # activate virtualenv source azEnv/bin/activate @@ -31,17 +43,41 @@ az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}" az account show # prepare running options -# base options -base_options="-c --no-exitfirst --report-path ./reports --reruns 3 --capture=sys" +# pytest options +pytest_options="--report-path ./reports" # parallel -if [[ ${PARALLELISM} -ge 2 ]]; then - base_options+=" -j ${PARALLELISM}" +if [[ -n ${PARALLELISM} ]]; then + pytest_options+=" -j ${PARALLELISM}" else - base_options+=" -s" + pytest_options+=" -s" +fi +# reruns +if [[ -n ${RERUNS} ]]; then + pytest_options+=" --reruns ${RERUNS}" +fi +# non boxed +if [[ -n ${NON_BOXED} ]]; then + pytest_options+=" --non-boxed" +fi +# log level +if [[ -n ${LOG_LEVEL} ]]; then + pytest_options+=" --log-level ${LOG_LEVEL}" +fi +# capture +if [[ -n ${CAPTURE} ]]; then + pytest_options+=" --capture ${CAPTURE}" +fi +# no exit first +if [[ -n ${NO_EXIT_FIRST} ]]; then + pytest_options+=" --no-exitfirst" +fi +# last failed +if [[ -n ${LAST_FAILED} ]]; then + pytest_options+=" --last-failed" fi # filter options -filter_options="" +filter_options=" -c" # cli matrix if [[ -n ${CLI_TEST_MATRIX} ]]; then filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}" @@ -55,11 +91,30 @@ if [[ -n ${CLI_TEST_COVERAGE} ]]; then filter_options+=" -cc ${CLI_TEST_COVERAGE}" fi +# case selection +selection_options="" +# enable selection +if [[ -n ${ENABLE_SELECTION} ]]; then + selection_options+=" --enable-selection" +fi +# selection mode +if [[ -n ${SELECTION_MODE} ]]; then + selection_options+=" --selection-mode ${SELECTION_MODE}" +fi +# selection count +if [[ -n ${SELECTION_COUNT} ]]; then + selection_options+=" --selection-count ${SELECTION_COUNT}" +fi +# unfix random seed +if [[ -n ${UNFIX_RANDOM_SEED} ]]; then + selection_options+=" --unfix-random-seed" +fi + # recording test if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then echo "Test in recording mode!" coverage_options=" -cc test_aks_commands.AzureKubernetesServiceScenarioTest" - recording_options="${base_options}${coverage_options}" + recording_options="${pytest_options}${coverage_options}${selection_options}" recording_options+=" --json-report-file=cli_recording_report.json" recording_options+=" --xml-file=cli_recording_result.xml" echo "Recording test options: ${recording_options}" @@ -67,8 +122,10 @@ if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then azaks ${recording_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Recording test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Recording test passed!\n\n" fi @@ -77,7 +134,7 @@ fi # live test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Test in live mode!" - live_options="${base_options}${filter_options}" + live_options="${pytest_options}${filter_options}${selection_options}" live_options+=" -l --json-report-file=cli_live_report.json" live_options+=" --xml-file=cli_live_result.xml" echo "Live test options: ${live_options}" @@ -85,8 +142,10 @@ if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then azaks ${live_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Live test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Live test passed!\n\n" fi @@ -95,7 +154,7 @@ fi # live-replay test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Test in live-replay mode!" - live_replay_options="${base_options}${filter_options}" + live_replay_options="${pytest_options}${filter_options}${selection_options}" live_replay_options+=" --json-report-file=cli_live_replay_report.json" live_replay_options+=" --xml-file=cli_live_replay_result.xml" echo "Live-replay test options: ${live_replay_options}" @@ -103,8 +162,10 @@ if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then azaks ${live_replay_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Live-replay test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Live-replay test passed!\n\n" fi diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh index c24fa1a09cb..37e799f9f08 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh @@ -6,8 +6,9 @@ set -o nounset set -o pipefail set -o xtrace -# const -acs_base_dir="azure-cli/src/azure-cli/azure/cli/command_modules/acs" +# check var +[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) +[[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1) # activate virtualenv source azEnv/bin/activate @@ -18,7 +19,7 @@ removeAKSPreview # unit test & coverage report acs_unit_test_failed="" -pushd ${acs_base_dir} +pushd ${ACS_BASE_DIR} # clean existing coverage report (coverage combine || true) && (coverage erase || true) # perform unit test with module 'unittest' @@ -32,10 +33,10 @@ coverage combine coverage report -m coverage json -o coverage_acs.json popd -mkdir -p reports/ && cp ${acs_base_dir}/coverage_acs.json reports/ +mkdir -p reports/ && cp ${ACS_BASE_DIR}/coverage_acs.json reports/ if [[ ${acs_unit_test_failed} == "true" ]]; then echo "Unit test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." exit 1 fi diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh index d33c0923e07..955168e423e 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh @@ -7,12 +7,20 @@ set -o pipefail set -o xtrace # check var +[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) [[ -z "${TENANT_ID}" ]] && (echo "TENANT_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_SUBSCRIPTION_ID}" ]] && (echo "AZCLI_ALT_SUBSCRIPTION_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_CLIENT_ID}" ]] && (echo "AZCLI_ALT_CLIENT_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "AZCLI_ALT_CLIENT_SECRET is empty"; exit 1) +[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty") [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1) -[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1) +[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty") +[[ -z "${RERUNS}" ]] && (echo "RERUNS is empty") +[[ -z "${NON_BOXED}" ]] && (echo "NON_BOXED is empty") +[[ -z "${LOG_LEVEL}" ]] && (echo "LOG_LEVEL is empty") +[[ -z "${CAPTURE}" ]] && (echo "CAPTURE is empty") +[[ -z "${NO_EXIT_FIRST}" ]] && (echo "NO_EXIT_FIRST is empty") +[[ -z "${LAST_FAILED}" ]] && (echo "LAST_FAILED is empty") [[ -z "${EXT_TEST_MATRIX}" ]] && (echo "EXT_TEST_MATRIX is empty") [[ -z "${EXT_TEST_FILTER}" ]] && (echo "EXT_TEST_FILTER is empty") [[ -z "${EXT_TEST_COVERAGE}" ]] && (echo "EXT_TEST_COVERAGE is empty") @@ -20,6 +28,10 @@ set -o xtrace [[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty") [[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty") [[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty") +[[ -z "${ENABLE_SELECTION}" ]] && (echo "ENABLE_SELECTION is empty") +[[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty") +[[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty") +[[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty") # activate virtualenv source azEnv/bin/activate @@ -35,17 +47,41 @@ az account show if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "only" ]]; then # prepare running options - # base options - base_options="-e --no-exitfirst --report-path ./reports --reruns 3 --capture=sys" + # pytest options + pytest_options="--report-path ./reports" # parallel - if [[ ${PARALLELISM} -ge 2 ]]; then - base_options+=" -j ${PARALLELISM}" + if [[ -n ${PARALLELISM} ]]; then + pytest_options+=" -j ${PARALLELISM}" else - base_options+=" -s" + pytest_options+=" -s" + fi + # reruns + if [[ -n ${RERUNS} ]]; then + pytest_options+=" --reruns ${RERUNS}" + fi + # non boxed + if [[ -n ${NON_BOXED} ]]; then + pytest_options+=" --non-boxed" + fi + # log level + if [[ -n ${LOG_LEVEL} ]]; then + pytest_options+=" --log-level ${LOG_LEVEL}" + fi + # capture + if [[ -n ${CAPTURE} ]]; then + pytest_options+=" --capture ${CAPTURE}" + fi + # no exit first + if [[ -n ${NO_EXIT_FIRST} ]]; then + pytest_options+=" --no-exitfirst" + fi + # last failed + if [[ -n ${LAST_FAILED} ]]; then + pytest_options+=" --last-failed" fi # filter options - filter_options="" + filter_options=" -e" # ext matrix if [[ -n ${EXT_TEST_MATRIX} ]]; then filter_options+=" -em ./configs/${EXT_TEST_MATRIX}" @@ -59,11 +95,30 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on filter_options+=" -ec ${EXT_TEST_COVERAGE}" fi + # case selection + selection_options="" + # enable selection + if [[ -n ${ENABLE_SELECTION} ]]; then + selection_options+=" --enable-selection" + fi + # selection mode + if [[ -n ${SELECTION_MODE} ]]; then + selection_options+=" --selection-mode ${SELECTION_MODE}" + fi + # selection count + if [[ -n ${SELECTION_COUNT} ]]; then + selection_options+=" --selection-count ${SELECTION_COUNT}" + fi + # unfix random seed + if [[ -n ${UNFIX_RANDOM_SEED} ]]; then + selection_options+=" --unfix-random-seed" + fi + # recording test if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then echo "Test in recording mode!" coverage_options=" -ec test_aks_commands.AzureKubernetesServiceScenarioTest" - recording_options="${base_options}${coverage_options}" + recording_options="${pytest_options}${coverage_options}${selection_options}" recording_options+=" --json-report-file=ext_recording_report.json" recording_options+=" --xml-file=ext_recording_result.xml" echo "Recording test options: ${recording_options}" @@ -71,8 +126,10 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on azaks ${recording_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Recording test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Recording test passed!\n\n" fi @@ -81,7 +138,7 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on # live test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Test in live mode!" - live_options="${base_options}${filter_options}" + live_options="${pytest_options}${filter_options}${selection_options}" live_options+=" -l --json-report-file=ext_live_report.json" live_options+=" --xml-file=ext_live_result.xml" echo "Live test options: ${live_options}" @@ -89,8 +146,10 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on azaks ${live_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Live test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Live test passed!\n\n" fi @@ -99,7 +158,7 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on # live-replay test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Test in live-replay mode!" - live_replay_options="${base_options}${filter_options}" + live_replay_options="${pytest_options}${filter_options}${selection_options}" live_replay_options+=" --json-report-file=ext_live_replay_report.json" live_replay_options+=" --xml-file=ext_live_replay_result.xml" echo "Live-replay test options: ${live_replay_options}" @@ -107,8 +166,10 @@ if [[ -z ${BACKWARD_COMPATIBILITY_TEST} || ${BACKWARD_COMPATIBILITY_TEST} != "on azaks ${live_replay_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Live-replay test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Live-replay test passed!\n\n" fi @@ -117,17 +178,41 @@ fi if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then # prepare running options - # base options - base_options="-c --no-exitfirst --report-path ./reports --reruns 3 --capture=sys" + # pytest options + pytest_options="--report-path ./reports" # parallel - if [[ ${PARALLELISM} -ge 2 ]]; then - base_options+=" -j ${PARALLELISM}" + if [[ -n ${PARALLELISM} ]]; then + pytest_options+=" -j ${PARALLELISM}" else - base_options+=" -s" + pytest_options+=" -s" + fi + # reruns + if [[ -n ${RERUNS} ]]; then + pytest_options+=" --reruns ${RERUNS}" + fi + # non boxed + if [[ -n ${NON_BOXED} ]]; then + pytest_options+=" --non-boxed" + fi + # log level + if [[ -n ${LOG_LEVEL} ]]; then + pytest_options+=" --log-level ${LOG_LEVEL}" + fi + # capture + if [[ -n ${CAPTURE} ]]; then + pytest_options+=" --capture ${CAPTURE}" + fi + # no exit first + if [[ -n ${NO_EXIT_FIRST} ]]; then + pytest_options+=" --no-exitfirst" + fi + # last failed + if [[ -n ${LAST_FAILED} ]]; then + pytest_options+=" --last-failed" fi # filter options - filter_options="" + filter_options=" -c" # cli matrix if [[ -n ${CLI_TEST_MATRIX} ]]; then filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}" @@ -141,10 +226,29 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then filter_options+=" -cc ${CLI_TEST_COVERAGE}" fi + # case selection + selection_options="" + # enable selection + if [[ -n ${ENABLE_SELECTION} ]]; then + selection_options+=" --enable-selection" + fi + # selection mode + if [[ -n ${SELECTION_MODE} ]]; then + selection_options+=" --selection-mode ${SELECTION_MODE}" + fi + # selection count + if [[ -n ${SELECTION_COUNT} ]]; then + selection_options+=" --selection-count ${SELECTION_COUNT}" + fi + # unfix random seed + if [[ -n ${UNFIX_RANDOM_SEED} ]]; then + selection_options+=" --unfix-random-seed" + fi + # live test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Backward Compatibility Test in live mode!" - live_options="${base_options}${filter_options}" + live_options="${pytest_options}${filter_options}${selection_options}" live_options+=" -l --json-report-file=bc_live_report.json" live_options+=" --xml-file=bc_live_result.xml" echo "Live test options: ${live_options}" @@ -152,8 +256,10 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then azaks ${live_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Backward Compatibility Live test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Backward Compatibility Live test passed!\n\n" fi @@ -162,7 +268,7 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then # live-replay test if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then echo "Backward Compatibility Test in live-replay mode!" - live_replay_options="${base_options}${filter_options}" + live_replay_options="${pytest_options}${filter_options}${selection_options}" live_replay_options+=" --json-report-file=bc_live_replay_report.json" live_replay_options+=" --xml-file=bc_live_replay_result.xml" echo "Live-replay test options: ${live_replay_options}" @@ -170,8 +276,10 @@ if [[ -n ${BACKWARD_COMPATIBILITY_TEST} ]]; then azaks ${live_replay_options} || test_result=$? if [[ ${test_result} -ne 0 ]]; then echo "Backward Compatibility Live-replay test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." - exit ${test_result} + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." + if [[ -z ${IGNORE_EXIT_CODE} ]]; then + exit ${test_result} + fi else echo -e "Backward Compatibility Live-replay test passed!\n\n" fi diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh index c4445141221..82298d71376 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh @@ -6,8 +6,9 @@ set -o nounset set -o pipefail set -o xtrace -# const -aks_preview_base_dir="azure-cli-extensions/src/aks-preview/azext_aks_preview" +# check var +[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) +[[ -z "${AKS_PREVIEW_BASE_DIR}" ]] && (echo "AKS_PREVIEW_BASE_DIR is empty"; exit 1) # activate virtualenv source azEnv/bin/activate @@ -17,7 +18,7 @@ source azEnv/bin/activate # unit test & coverage report azext_aks_preview_unit_test_failed="" -pushd ${aks_preview_base_dir} +pushd ${AKS_PREVIEW_BASE_DIR} # clean existing coverage report (coverage combine || true) && (coverage erase || true) # perform unit test with module 'unittest' @@ -31,10 +32,10 @@ coverage combine coverage report -m coverage json -o coverage_azext_aks_preview.json popd -mkdir -p reports/ && cp ${aks_preview_base_dir}/coverage_azext_aks_preview.json reports/ +mkdir -p reports/ && cp ${AKS_PREVIEW_BASE_DIR}/coverage_azext_aks_preview.json reports/ if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then echo "Unit test failed!" - echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines." + echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines." exit 1 fi diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh deleted file mode 100644 index 8f132ac633b..00000000000 --- a/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# bash options -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace diff --git a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh index 19eb68518a5..62275fcc9a5 100755 --- a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh +++ b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh @@ -7,39 +7,56 @@ set -o pipefail set -o xtrace # check var -# from variable group -# client id, secret and tenant id of sp 'azcli-aks-live-test' +# azure devops +[[ -z "${BUILD_REASON}" ]] && (echo "BUILD_REASON is empty") +[[ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ]] && (echo "SYSTEM_PULLREQUEST_TARGETBRANCH is empty") + +# client id, secret and tenant id of sp 'azcli-aks-live-test' (from variable group "azcli-aks-tool") [[ -z "${AZCLI_ALT_CLIENT_ID}" ]] && (echo "AZCLI_ALT_CLIENT_ID is empty"; exit 1) [[ -z "${AZCLI_ALT_SUBSCRIPTION_ID}" ]] && (echo "AZCLI_ALT_SUBSCRIPTION_ID is empty"; exit 1) [[ -z "${TENANT_ID}" ]] && (echo "TENANT_ID is empty"; exit 1) -# basic ubuntu image with python3 pre-installed, hosted on some container registry + +# basic ubuntu image with python3 pre-installed, hosted on some container registry (from variable group "azcli-aks-tool") [[ -z "${IMAGE_PREFIX}" ]] && (echo "IMAGE_PREFIX is empty"; exit 1) [[ -z "${IMAGE_NAME}" ]] && (echo "IMAGE_NAME is empty"; exit 1) [[ -z "${IMAGE_TAG}" ]] && (echo "IMAGE_TAG is empty"; exit 1) + # specify the version of python3 [[ -z "${PYTHON_VERSION}" ]] && (echo "PYTHON_VERSION is empty"; exit 1) -# from azure devops -[[ -z "${BUILD_REASON}" ]] && (echo "BUILD_REASON is empty") -[[ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ]] && (echo "SYSTEM_PULLREQUEST_TARGETBRANCH is empty") +# tsg/wiki link +[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1) + +# pytest options +[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty") +[[ -z "${RERUNS}" ]] && (echo "RERUNS is empty") +[[ -z "${NON_BOXED}" ]] && (echo "NON_BOXED is empty") +[[ -z "${LOG_LEVEL}" ]] && (echo "LOG_LEVEL is empty") +[[ -z "${CAPTURE}" ]] && (echo "CAPTURE is empty") +[[ -z "${NO_EXIT_FIRST}" ]] && (echo "NO_EXIT_FIRST is empty") +[[ -z "${LAST_FAILED}" ]] && (echo "LAST_FAILED is empty") # custom +[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty") [[ -z "${TEST_LOCATION}" ]] && (echo "TEST_LOCATION is empty"; exit 1) [[ -z "${COVERAGE}" ]] && (echo "COVERAGE is empty"; exit 1) [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1) -[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1) -[[ -z "${TEST_CASES}" ]] && (echo "TEST_CASES is empty") [[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty") [[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty") [[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty") [[ -z "${EXT_TEST_MATRIX}" ]] && (echo "EXT_TEST_MATRIX is empty") [[ -z "${EXT_TEST_FILTER}" ]] && (echo "EXT_TEST_FILTER is empty") [[ -z "${EXT_TEST_COVERAGE}" ]] && (echo "EXT_TEST_COVERAGE is empty") +[[ -z "${ENABLE_SELECTION}" ]] && (echo "ENABLE_SELECTION is empty") +[[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty") +[[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty") +[[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty") +[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty") [[ -z "${CLI_REPO}" ]] && (echo "CLI_REPO is empty"; exit 1) [[ -z "${CLI_BRANCH}" ]] && (echo "CLI_BRANCH is empty"; exit 1) [[ -z "${EXT_REPO}" ]] && (echo "EXT_REPO is empty"; exit 1) [[ -z "${EXT_BRANCH}" ]] && (echo "EXT_BRANCH is empty"; exit 1) -[[ -z "${BACKWARD_COMPATIBILITY_TEST}" ]] && (echo "BACKWARD_COMPATIBILITY_TEST is empty") + # base directories for acs, aks-preview and live test [[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1) [[ -z "${AKS_PREVIEW_BASE_DIR}" ]] && (echo "AKS_PREVIEW_BASE_DIR is empty"; exit 1) @@ -48,6 +65,10 @@ set -o xtrace # clear cat /dev/null > env.list +# azure devops +echo "BUILD_REASON=${BUILD_REASON}" >> env.list +echo "SYSTEM_PULLREQUEST_TARGETBRANCH=${SYSTEM_PULLREQUEST_TARGETBRANCH}" >> env.list + # tenant, sub, client echo "TENANT_ID=${TENANT_ID}" >> env.list echo "AZCLI_ALT_SUBSCRIPTION_ID=${AZCLI_ALT_SUBSCRIPTION_ID}" >> env.list @@ -58,39 +79,50 @@ echo "IMAGE_PREFIX=${IMAGE_PREFIX}" >> env.list echo "IMAGE_NAME=${IMAGE_NAME}" >> env.list echo "IMAGE_TAG=${IMAGE_TAG}" >> env.list -# predefined variables -echo "BUILD_REASON=${BUILD_REASON}" >> env.list -echo "SYSTEM_PULLREQUEST_TARGETBRANCH=${SYSTEM_PULLREQUEST_TARGETBRANCH}" >> env.list - # python version echo "PYTHON_VERSION=${PYTHON_VERSION}" >> env.list -# base directories -echo "ACS_BASE_DIR=${ACS_BASE_DIR}" >> env.list -echo "AKS_PREVIEW_BASE_DIR=${AKS_PREVIEW_BASE_DIR}" >> env.list -echo "LIVE_TEST_BASE_DIR=${LIVE_TEST_BASE_DIR}" >> env.list +# tsg/wiki link +echo "WIKI_LINK=${WIKI_LINK}" >> env.list + +# pytest options +echo "PARALLELISM=${PARALLELISM}" >> env.list +echo "RERUNS=${RERUNS}" >> env.list +echo "NON_BOXED=${NON_BOXED}" >> env.list +echo "LOG_LEVEL=${LOG_LEVEL}" >> env.list +echo "CAPTURE=${CAPTURE}" >> env.list +echo "NO_EXIT_FIRST=${NO_EXIT_FIRST}" >> env.list +echo "LAST_FAILED=${LAST_FAILED}" >> env.list -# azdev env +# custom - azdev env echo "AZURE_CLI_TEST_DEV_SP_NAME=${AZCLI_ALT_CLIENT_ID}" >> env.list echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=${TEST_LOCATION}" >> env.list -# test +# custom - az-aks-tool +echo "IGNORE_EXIT_CODE=${IGNORE_EXIT_CODE}" >> env.list echo "COVERAGE=${COVERAGE}" >> env.list echo "TEST_MODE=${TEST_MODE}" >> env.list -echo "PARALLELISM=${PARALLELISM}" >> env.list -echo "TEST_CASES=${TEST_CASES}" >> env.list echo "CLI_TEST_MATRIX=${CLI_TEST_MATRIX}" >> env.list echo "CLI_TEST_FILTER=${CLI_TEST_FILTER}" >> env.list echo "CLI_TEST_COVERAGE=${CLI_TEST_COVERAGE}" >> env.list echo "EXT_TEST_MATRIX=${EXT_TEST_MATRIX}" >> env.list echo "EXT_TEST_FILTER=${EXT_TEST_FILTER}" >> env.list echo "EXT_TEST_COVERAGE=${EXT_TEST_COVERAGE}" >> env.list +echo "ENABLE_SELECTION=${ENABLE_SELECTION}" >> env.list +echo "SELECTION_MODE=${SELECTION_MODE}" >> env.list +echo "SELECTION_COUNT=${SELECTION_COUNT}" >> env.list +echo "UNFIX_RANDOM_SEED=${UNFIX_RANDOM_SEED}" >> env.list -# repo +# custom - repo echo "CLI_REPO=${CLI_REPO}" >> env.list echo "CLI_BRANCH=${CLI_BRANCH}" >> env.list echo "EXT_REPO=${EXT_REPO}" >> env.list echo "EXT_BRANCH=${EXT_BRANCH}" >> env.list -# backward compatibility -echo "BACKWARD_COMPATIBILITY_TEST=${BACKWARD_COMPATIBILITY_TEST}" >> env.list \ No newline at end of file +# custom - backward compatibility +echo "BACKWARD_COMPATIBILITY_TEST=${BACKWARD_COMPATIBILITY_TEST}" >> env.list + +# base directories +echo "ACS_BASE_DIR=${ACS_BASE_DIR}" >> env.list +echo "AKS_PREVIEW_BASE_DIR=${AKS_PREVIEW_BASE_DIR}" >> env.list +echo "LIVE_TEST_BASE_DIR=${LIVE_TEST_BASE_DIR}" >> env.list