Basic script to get command coverage (Azure#245)

Basic script to get command coverage

.gitignore

@@ -59,3 +59,6 @@ src/build
 
 # Azure deployment credentials
 *.pubxml
+
+# Auxiliary files
+command_coverage.txt

scripts/command_coverage.py

@@ -0,0 +1,53 @@
+from __future__ import print_function
+import os
+import sys
+from subprocess import check_call, CalledProcessError
+import azure.cli.application as application
+
+COVERAGE_FILE = 'command_coverage.txt'
+DEVNULL = open(os.devnull, 'w')
+
+config = application.Configuration([])
+application.APPLICATION = application.Application(config)
+cmd_table = config.get_command_table()
+cmd_list = [x['name'].strip() for x in cmd_table.values()]
+cmd_set = set(cmd_list)
+if os.path.isfile(COVERAGE_FILE):
+    os.remove(COVERAGE_FILE)
+
+print('Running tests...')
+try:
+    check_call(['python', 'scripts/command_modules/test.py'], stdout=DEVNULL, stderr=DEVNULL,
+               env=dict(os.environ, AZURE_CLI_TEST_TRACK_COMMANDS='1'))
+except CalledProcessError as err:
+    print(err, file=sys.stderr)
+    print("Tests failed.")
+    sys.exit(1)
+print('Tests passed.')
+
+commands_tested_with_params = [line.rstrip('\n') for line in open(COVERAGE_FILE)]
+
+commands_tested = []
+for tested_command in commands_tested_with_params:
+    for c in cmd_list:
+        if tested_command.startswith(c):
+            commands_tested.append(c)
+
+commands_tested_set = set(commands_tested)
+untested = list(cmd_set - commands_tested_set)
+print()
+print("Untested commands")
+print("=================")
+print('\n'.join(sorted(untested)))
+percentage_tested = (len(commands_tested_set) / len(cmd_set)) * 100
+print()
+print('Total commands {}, Tested commands {}, Untested commands {}'.format(
+    len(cmd_set),
+    len(commands_tested_set),
+    len(cmd_set)-len(commands_tested_set)))
+print('COMMAND COVERAGE {0:.2f}%'.format(percentage_tested))
+
+# Delete the command coverage file
+if os.path.isfile(COVERAGE_FILE):
+    os.remove(COVERAGE_FILE)
+

src/azure/cli/utils/command_test_script.py

@@ -11,6 +11,9 @@
 from azure.cli.parser import IncorrectUsageError
 from azure.cli._util import CLIError
 
+TRACK_COMMANDS = os.environ.get('AZURE_CLI_TEST_TRACK_COMMANDS')
+COMMAND_COVERAGE_FILENAME = 'command_coverage.txt'
+
 class JMESPathComparatorAssertionError(AssertionError):
 
     def __init__(self, comparator, actual_result, json_data):
@@ -69,12 +72,16 @@ def __init__(self, set_up, test_body, tear_down):
         else:
             raise TypeError('test_body must be callable')
         self.tear_down = tear_down
+        self.track_commands = False
 
     def run_test(self):
         try:
             if hasattr(self.set_up, '__call__'):
                 self.set_up()
+            # only track commands for the test body
+            self.track_commands = TRACK_COMMANDS
             self.test_body()
+            self.track_commands = False
         except Exception: #pylint: disable=broad-except
             traceback.print_exc(file=self._display)
             self.fail = True
@@ -86,6 +93,14 @@ def run_test(self):
             self._display.close()
             self._raw.close()
 
+    def _track_executed_commands(self, command):
+        if not self.track_commands:
+            return
+        filename = COMMAND_COVERAGE_FILENAME
+        with open(filename, 'a+') as f:
+            f.write(' '.join(command))
+            f.write('\n')
+
     def rec(self, command):
         ''' Run a command and save the output as part of the expected results. This will also
         save the output to a display file so you can see the command, followed by its output
@@ -97,6 +112,7 @@ def rec(self, command):
         output = StringIO()
         command_list = command if isinstance(command, list) else command.split()
         cli(command_list, file=output)
+        self._track_executed_commands(command_list)
         result = output.getvalue().strip()
         self._display.write('\n\n== {} ==\n\n{}'.format(command, result))
         self._raw.write(result)
@@ -111,6 +127,7 @@ def run(self, command): #pylint: disable=no-self-use
         output = StringIO()
         command_list = command if isinstance(command, list) else command.split()
         cli(command_list, file=output)
+        self._track_executed_commands(command_list)
         result = output.getvalue().strip()
         output.close()
         return result
@@ -125,6 +142,7 @@ def test(self, command, checks):
         command_list = command if isinstance(command, list) else command.split()
         command_list += ['-o', 'json']
         cli(command_list, file=output)
+        self._track_executed_commands(command_list)
         result = output.getvalue().strip()
         self._raw.write(result)
         try: