Run module tests in parallel

.gitignore

@@ -57,6 +57,7 @@ src/build
 /doc/sphinx/_build
 /.vs/config/applicationhost.config
 .vscode/settings.json
+.vscode/.ropeproject/
 
 # Azure deployment credentials
 *.pubxml

scripts/command_modules/_common.py

@@ -6,8 +6,9 @@
 from __future__ import print_function
 import os
 import sys
+import subprocess
 
-from subprocess import check_call, CalledProcessError
+from subprocess import check_call, check_output, CalledProcessError
 
 COMMAND_MODULE_PREFIX = 'azure-cli-'
 
@@ -37,6 +38,20 @@ def exec_command(command, cwd=None, stdout=None, env=None):
         print(err, file=sys.stderr)
         return False
 
+def exec_command_output(command, env=None):
+    """
+    Execute a command and return its output as well as the status code.
+    """
+    try:
+        output = check_output(
+            command if isinstance(command, list) else command.split(),
+            env=os.environ.copy().update(env or {}),
+            stderr=subprocess.STDOUT)
+        return (output.decode('utf-8'), 0)
+    except CalledProcessError as err:
+        return (err.output, err.returncode)
+
+
 def print_summary(failed_modules):
     print()
     print("SUMMARY")

scripts/command_modules/test.py

@@ -6,37 +6,100 @@
 ## Run the tests for each command module ##
 
 from __future__ import print_function
+
 import os
 import sys
+from datetime import datetime
+from threading import Thread
+from _common import get_all_command_modules, exec_command_output, COMMAND_MODULE_PREFIX
+
+
+class TestTask(Thread):
+    LOG_DIR = os.path.expanduser(os.path.join('~', '.azure', 'logs'))
+
+    """
+    Execute a test task in a separated task.
+    """
+    def __init__(self, module_name, module_path):
+        Thread.__init__(self)
+        module_name = module_name.replace(COMMAND_MODULE_PREFIX, '')
+        self._path_to_module = os.path.join(
+            module_path, 'azure', 'cli', 'command_modules', module_name, 'tests')
+
+        self.name = module_name
+        self.skipped = not os.path.isdir(self._path_to_module)
+        self.rtcode = -1
+        self._started_on = None
+        self._finished_on = None
+
+        self.start()
+
+    def run(self):
+        if self.skipped:
+            return
+
+        command = 'python -m unittest discover -s {0}'.format(self._path_to_module)
+
+        if os.environ.get('CONTINUOUS_INTEGRATION') and os.environ.get('TRAVIS'):
+            command += " --buffer"
+
+        env = os.environ.copy()
+        env.update({'AZURE_CLI_ENABLE_LOG_FILE': '1', 'AZURE_CLI_LOG_DIR': TestTask.LOG_DIR})
+
+        print('Executing tests for module {0}'.format(self.name))
+
+        self._started_on = datetime.now()
+        output, self.rtcode = exec_command_output(command, env)
+        self._finished_on = datetime.now()
+
+        print('[{1}] Finish testing module {0}.'.format(
+            self.name, 'Passed' if self.rtcode == 0 else 'Failed'))
+
+        if self.rtcode != 0:
+            print(output)
+
+    def print_format(self, summary_format):
+        status = 'skipped' if self.skipped else ('failed' if self.rtcode != 0 else 'passed')
+
+        args = [self.name, status]
+        if self._started_on and self._finished_on: 
+            args.append(self._started_on.strftime('%H:%M:%S'))
+            args.append(str((self._finished_on - self._started_on).total_seconds()))
+        else:
+            args.append('')
+            args.append('')
+
+        print(summary_format.format(*args))
+
+
+def wait_all_tasks(tasks_list):
+    for each in tasks_list:
+        each.join()
 
-from _common import get_all_command_modules, exec_command, print_summary, COMMAND_MODULE_PREFIX
+def get_print_template(tasks_list):
+    max_module_name_len = max([len(t.name) for t in tasks_list])
 
-LOG_DIR = os.path.expanduser(os.path.join('~', '.azure', 'logs'))
+    # the format: <module name>  <status> <start on> <finish on>
+    return '{0:' + str(max_module_name_len + 2) + '}{1:10}{2:12}{3:10}'
 
-all_command_modules = get_all_command_modules()
-print("Running tests on command modules.")
+def run_module_tests():
+    print("Running tests on command modules.")
 
-failed_module_names = []
-skipped_modules = []
-for name, fullpath in all_command_modules:
-    path_to_module = os.path.join(fullpath, 'azure', 'cli', 'command_modules', name.replace(COMMAND_MODULE_PREFIX, ''), 'tests')
-    if not os.path.isdir(path_to_module):
-        skipped_modules.append(name)
-        continue
-    command = "python -m unittest discover -s " + path_to_module
-    # append --buffer when running on CI to ensure any unrecorded tests fail instead of hang
-    if os.environ.get('CONTINUOUS_INTEGRATION') and os.environ.get('TRAVIS'):
-        command += " --buffer"
-    success = exec_command(command, env={'AZURE_CLI_ENABLE_LOG_FILE': '1', 'AZURE_CLI_LOG_DIR': LOG_DIR})
-    if not success:
-        failed_module_names.append(name)
+    tasks = [TestTask(name, path) for name, path in get_all_command_modules()]
+    wait_all_tasks(tasks)
 
-print_summary(failed_module_names)
+    print()
+    print("Summary")
+    print("========================")
+    summary_format = get_print_template(tasks)
+    for t in tasks:
+        t.print_format(summary_format)
 
-print("Full debug log available at '{}'.".format(LOG_DIR))
+    print("========================")
+    print("* Full debug log available at '{}'.".format(TestTask.LOG_DIR))
 
-if failed_module_names:
-    sys.exit(1)
+    if any([task.name for task in tasks if task.rtcode != 0 and not task.skipped]):
+        sys.exit(1)
 
-if skipped_modules:
-    print("Modules skipped as no test dir found:", ', '.join(skipped_modules), file=sys.stderr)
+if __name__ == '__main__':
+    run_module_tests()