Better test discovery (#12292)

* use chip tool to figure out exactly what tests exist rather than searching for YAML files

* Fix arguments for running tests

* Restyle fixes

* Update to the test list command to finding out available tests

* ignore empty lines

* Update scripts/tests/run_test_suite.py

Co-authored-by: Victor Morales <chipahuac@hotmail.com>

* Add missing \

Co-authored-by: Justin Wood <woody@apple.com>
Co-authored-by: Victor Morales <chipahuac@hotmail.com>

.github/workflows/tests.yaml

@@ -87,9 +87,10 @@ jobs:
               timeout-minutes: 30
               run: |
                   ./scripts/run_in_build_env.sh \
-                  "./scripts/tests/run_test_suite.py run \
-                     --iterations 2 \
+                  "./scripts/tests/run_test_suite.py \
                      --chip-tool ./out/debug/standalone/chip-tool \
+                     run \
+                     --iterations 2 \
                      --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \
                      --tv-app ./out/debug/standalone/chip-tv-app \
                   "
@@ -176,10 +177,10 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \
+                     --chip-tool ./out/debug/standalone/chip-tool \
                      --target-skip-glob 'TV_*' \
                      run \
                      --iterations 2 \
-                     --chip-tool ./out/debug/standalone/chip-tool \
                      --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \
                      --tv-app ./out/debug/standalone/chip-tv-app \
                   "

scripts/tests/chiptest/__init__.py

@@ -17,34 +17,31 @@
 from pathlib import Path
 import os
 import logging
+import subprocess
+import re
 
 import chiptest.linux
 import chiptest.runner
 
 from .test_definition import TestTarget, TestDefinition, ApplicationPaths
 
 
-def AllTests(root: str):
-    """Gets all the tests that can be found in the ROOT directory based on
-       yaml file names.
+def AllTests(chip_tool: str):
+    """Executes `chip_tool` binary to see what tests are available.
     """
-    for path in Path(os.path.join(root, 'src', 'app',  'tests', 'suites')).rglob("*.yaml"):
-        logging.debug('Found YAML: %s' % path)
 
-        # grab the name without the extension
-        name = path.stem
+    result = subprocess.run([chip_tool, 'tests', 'list'], capture_output=True)
 
-        if 'Simulated' in name:
+    for name in result.stdout.decode('utf8').split('\n'):
+        if not name:
             continue
 
         if name.startswith('TV_'):
             target = TestTarget.TV
-        elif name.startswith('Test'):
-            target = TestTarget.ALL_CLUSTERS
         else:
-            continue
+            target = TestTarget.ALL_CLUSTERS
 
-        yield TestDefinition(yaml_file=path, run_name=path.stem, name=name, target=target)
+        yield TestDefinition(run_name=name, name=name, target=target)
 
 
 __all__ = ['TestTarget', 'TestDefinition', 'AllTests', 'ApplicationPaths']

scripts/tests/chiptest/test_definition.py

@@ -79,7 +79,6 @@ def LogContents(self):
 
 @dataclass
 class TestDefinition:
-    yaml_file: str
     name: str
     run_name: str
     target: TestTarget

scripts/tests/run_test_suite.py

@@ -61,6 +61,7 @@ class RunContext:
     root: str
     tests: typing.List[chiptest.TestDefinition]
     in_unshare: bool
+    chip_tool: str
 
 
 @click.group(chain=True)
@@ -93,24 +94,28 @@ class RunContext:
 @click.option(
     '--root',
     default=DEFAULT_CHIP_ROOT,
-    help='Default directory path for CHIP. Used to determine what tests exist')
+    help='Default directory path for CHIP. Used to copy run configurations')
 @click.option(
     '--internal-inside-unshare',
     hidden=True,
     is_flag=True,
     default=False,
     help='Internal flag for running inside a unshared environment'
 )
+@click.option(
+    '--chip-tool',
+    default=FindBinaryPath('chip-tool'),
+    help='Binary path of chip tool app to use to run the test')
 @click.pass_context
-def main(context, log_level, target, target_glob, target_skip_glob, no_log_timestamps, root, internal_inside_unshare):
+def main(context, log_level, target, target_glob, target_skip_glob, no_log_timestamps, root, internal_inside_unshare, chip_tool):
     # Ensures somewhat pretty logging of what is going on
     log_fmt = '%(asctime)s.%(msecs)03d %(levelname)-7s %(message)s'
     if no_log_timestamps:
         log_fmt = '%(levelname)-7s %(message)s'
     coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
 
     # Figures out selected test that match the given name(s)
-    all_tests = [test for test in chiptest.AllTests(root)]
+    all_tests = [test for test in chiptest.AllTests(chip_tool)]
     tests = all_tests
     if 'all' not in target:
         tests = []
@@ -139,7 +144,8 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times
     tests.sort(key=lambda x: x.name)
 
     context.obj = RunContext(root=root, tests=tests,
-                             in_unshare=internal_inside_unshare)
+                             in_unshare=internal_inside_unshare,
+                             chip_tool=chip_tool)
 
 
 @main.command(
@@ -156,10 +162,6 @@ def cmd_generate(context):
     '--iterations',
     default=1,
     help='Number of iterations to run')
-@click.option(
-    '--chip-tool',
-    default=FindBinaryPath('chip-tool'),
-    help='What chip tool app to use to run the test')
 @click.option(
     '--all-clusters-app',
     default=FindBinaryPath('chip-all-clusters-app'),
@@ -169,12 +171,12 @@ def cmd_generate(context):
     default=FindBinaryPath('chip-tv-app'),
     help='what tv app to use')
 @click.pass_context
-def cmd_run(context, iterations, chip_tool, all_clusters_app, tv_app):
+def cmd_run(context, iterations, all_clusters_app, tv_app):
     runner = chiptest.runner.Runner()
 
     # Command execution requires an array
     paths = chiptest.ApplicationPaths(
-        chip_tool=[chip_tool],
+        chip_tool=[context.obj.chip_tool],
         all_clusters_app=[all_clusters_app],
         tv_app=[tv_app]
     )