"run_tests" script enhancements (Azure#5468)

* Test script enhancements.

* Code review feedback

* Update travis.yml file.

* Update .travis.yml config.

.travis.yml

@@ -23,11 +23,11 @@ jobs:
       python: 3.6
     - stage: verify
       env: PURPOSE='Automation'
-      script: ./scripts/ci/test_automation.sh
+      script: travis_wait 30 ./scripts/ci/test_automation.sh
       python: 3.6
     - stage: verify
       env: PURPOSE='Automation'
-      script: ./scripts/ci/test_automation.sh
+      script: travis_wait 30 ./scripts/ci/test_automation.sh
       python: 2.7
     - stage: verify
       script: ./scripts/ci/test_static.sh

azure-cli2017.pyproj

@@ -64,7 +64,7 @@
     <Compile Include="azure-cli-core\azure\cli\core\tests\test_storage_validators.py" />
     <Compile Include="azure-cli-core\azure\cli\core\tests\test_telemetry.py" />
     <Compile Include="azure-cli-core\azure\cli\core\tests\test_util.py" />
-    <Compile Include="azure-cli-core\azure\cli\core\tests\test_validators.py" />
+    <Compile Include="azure-cli-core\azure\cli\core\tests\test_core_validators.py" />
     <Compile Include="azure-cli-core\azure\cli\core\util.py" />
     <Compile Include="azure-cli-core\azure\cli\core\_config.py" />
     <Compile Include="azure-cli-core\azure\cli\core\_debug.py" />
@@ -395,7 +395,7 @@
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\commands.py" />
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\custom.py" />
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\tests\latest\test_lab.py" />
-    <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\tests\latest\test_validators.py" />
+    <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\tests\latest\test_lab_validators.py" />
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\validators.py" />
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\_client_factory.py" />
     <Compile Include="command_modules\azure-cli-lab\azure\cli\command_modules\lab\_format.py" />
@@ -448,7 +448,7 @@
     <Compile Include="command_modules\azure-cli-network\azure\cli\command_modules\network\_template_builder.py" />
     <Compile Include="command_modules\azure-cli-network\azure\cli\command_modules\network\_util.py" />
     <Compile Include="command_modules\azure-cli-network\azure\cli\command_modules\network\_validators.py" />
-    <Compile Include="command_modules\azure-cli-profile\azure\cli\command_modules\profile\tests\latest\test_custom.py" />
+    <Compile Include="command_modules\azure-cli-profile\azure\cli\command_modules\profile\tests\latest\test_profile_custom.py" />
     <Compile Include="command_modules\azure-cli-profile\azure\cli\command_modules\profile\_completers.py" />
     <Compile Include="command_modules\azure-cli-profile\azure\cli\command_modules\profile\_format.py" />
     <Compile Include="command_modules\azure-cli-profile\azure\cli\command_modules\profile\_help.py">
@@ -488,7 +488,7 @@
     <Compile Include="command_modules\azure-cli-reservations\azure_bdist_wheel.py" />
     <Compile Include="command_modules\azure-cli-reservations\setup.py" />
     <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_api_check.py" />
-    <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_custom.py" />
+    <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_resource_custom.py" />
     <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_locks.py" />
     <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_resource.py" />
     <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test_resource_id_deletes.py" />
@@ -502,7 +502,7 @@
     <Compile Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\_params.py" />
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\custom.py" />
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\role_scenario_test.py" />
-    <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\test_custom.py" />
+    <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\test_role_custom.py" />
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\test_graph.py" />
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\test_role.py" />
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\latest\test_role_commands_thru_mock.py" />

scripts/ci/test_automation.sh

@@ -41,7 +41,7 @@ title "Switch profile to $target_profile"
 
 az cloud set -n AzureCloud --profile $target_profile
 
-title 'Runing tests'
-coverage run -m automation test --parallel --ci
+title 'Running tests'
+coverage run -m automation test --ci
 coverage combine
 codecov

src/azure-cli-core/azure/cli/core/tests/test_generic_update.py

@@ -97,7 +97,7 @@ class GenericUpdateTest(unittest.TestCase):
     def setUpClass(cls):
         logging.getLogger().setLevel(logging.ERROR)
 
-    def test_generic_update(self):  # pylint: disable=too-many-statements
+    def test_generic_update_scenario(self):  # pylint: disable=too-many-statements
 
         my_obj, loader_cls = _prepare_test_loader()
         cli = TestCli(commands_loader_cls=loader_cls)

src/azure-cli-testsdk/azure/cli/testsdk/base.py

@@ -4,7 +4,6 @@
 # --------------------------------------------------------------------------------------------
 
 import os
-import subprocess
 import json
 import shlex
 import logging
@@ -164,26 +163,23 @@ def get_subscription_id(self):
 
 
 class ExecutionResult(object):
-    def __init__(self, cli_ctx, command, expect_failure=False, in_process=True):
+    def __init__(self, cli_ctx, command, expect_failure=False):
         self.output = ''
         self.applog = ''
+        self.command_coverage = {}
 
-        if in_process:
-            self._in_process_execute(cli_ctx, command, expect_failure=expect_failure)
-        else:
-            self._out_of_process_execute(command)
+        self._in_process_execute(cli_ctx, command, expect_failure=expect_failure)
 
+        log_val = ('Logging ' + self.applog) if self.applog else ''
         if expect_failure and self.exit_code == 0:
-            logger.error('Command "%s" => %d. (It did not fail as expected) Output: %s. %s', command,
-                         self.exit_code, self.output, ('Logging ' + self.applog) if self.applog else '')
+            logger.error('Command "%s" => %d. (It did not fail as expected). %s\n', command,
+                         self.exit_code, log_val)
             raise AssertionError('The command did not fail as it was expected.')
         elif not expect_failure and self.exit_code != 0:
-            logger.error('Command "%s" => %d. Output: %s. %s', command, self.exit_code, self.output,
-                         ('Logging ' + self.applog) if self.applog else '')
+            logger.error('Command "%s" => %d. %s\n', command, self.exit_code, log_val)
             raise AssertionError('The command failed. Exit code: {}'.format(self.exit_code))
 
-        logger.info('Command "%s" => %d. Output: %s. %s', command, self.exit_code, self.output,
-                    ('Logging ' + self.applog) if self.applog else '')
+        logger.info('Command "%s" => %d. %s\n', command, self.exit_code, log_val)
 
         self.json_value = None
         self.skip_assert = os.environ.get(ENV_SKIP_ASSERT, None) == 'True'
@@ -246,13 +242,5 @@ def _in_process_execute(self, cli_ctx, command, expect_failure=False):
             stdout_buf.close()
             logging_buf.close()
 
-    def _out_of_process_execute(self, command):
-        try:
-            self.output = subprocess.check_output(shlex.split(command)).decode('utf-8')
-            self.exit_code = 0
-        except subprocess.CalledProcessError as error:
-            self.exit_code, self.output = error.returncode, error.output.decode('utf-8')
-            self.process_error = error
-
 
 execute = ExecutionResult

src/command_modules/azure-cli-acr/azure/cli/command_modules/acr/tests/latest/test_acr_commands_mock.py

@@ -29,7 +29,7 @@
 from azure.cli.testsdk import TestCli
 
 
-class AcrCommandsTests(unittest.TestCase):
+class AcrMockCommandsTests(unittest.TestCase):
 
     @mock.patch('azure.cli.command_modules.acr.repository.get_access_credentials', autospec=True)
     @mock.patch('requests.get', autospec=True)

src/command_modules/azure-cli-monitor/azure/cli/command_modules/monitor/tests/latest/test_monitor_diagnostic_settings.py

@@ -10,7 +10,7 @@
 class TestMonitorDiagnosticSettings(ScenarioTest):
     @ResourceGroupPreparer(location='southcentralus')
     @StorageAccountPreparer(location='southcentralus')
-    def test_monitor_diagnostic_settings(self, resource_group, storage_account):
+    def test_monitor_diagnostic_settings_scenario(self, resource_group, storage_account):
         nsg_name = self.create_random_name(prefix='nsg', length=16)
         self.cmd('network nsg create -n {} -g {}'.format(nsg_name, resource_group))
 

src/command_modules/azure-cli-monitor/azure/cli/command_modules/monitor/tests/latest/test_monitor_metrics.py

@@ -9,7 +9,7 @@
 class TestMonitorMetrics(ScenarioTest):
     @ResourceGroupPreparer(location='southcentralus')
     @StorageAccountPreparer()
-    def test_monitor_metrics(self, resource_group, storage_account):
+    def test_monitor_metrics_scenario(self, resource_group, storage_account):
         resource_id = self.cmd(
             'az storage account show -n {} -g {} --query id -otsv'.format(storage_account, resource_group)).output