Add some options to dump the message expected by the Test Harness (#2…

…6054)

scripts/py_matter_yamltests/matter_yamltests/hooks.py

@@ -98,12 +98,15 @@ def stop(self, duration: int):
         """
         pass
 
-    def test_start(self, name: str, count: int):
+    def test_start(self, filename: str, name: str, count: int):
         """
         This method is called when the runner starts running a single test.
 
         Parameters
         ----------
+        filename: str
+            The name of the file containing the test that is starting.
+
         name: str
             The name of the test that is starting.
 
@@ -126,14 +129,17 @@ def test_stop(self, exception: Exception, duration: int):
         """
         pass
 
-    def step_skipped(self, name: str):
+    def step_skipped(self, name: str, expression: str):
         """
         This method is called when running a step is skipped.
 
         Parameters
         ----------
         name: str
             The name of the test step that is skipped.
+
+        expression: str
+            The PICS expression that results in the test step being skipped.
         """
         pass
 
@@ -148,7 +154,7 @@ def step_start(self, name: str):
         """
         pass
 
-    def step_success(self, logger, logs, duration: int):
+    def step_success(self, logger, logs, duration: int, request):
         """
         This method is called when running a step succeeds.
 
@@ -162,10 +168,13 @@ def step_success(self, logger, logs, duration: int):
 
         duration: int
             How long it took to run the test step, in milliseconds.
+
+        request:
+            The original request as defined by the test step.
         """
         pass
 
-    def step_failure(self, logger, logs, duration: int, expected, received):
+    def step_failure(self, logger, logs, duration: int, request, received):
         """
         This method is called when running a step fails.
 
@@ -180,8 +189,8 @@ def step_failure(self, logger, logs, duration: int, expected, received):
         duration: int
             How long it took to run the test step, in milliseconds.
 
-        expected:
-            The expected response as defined by the test step.
+        request:
+            The original request as defined by the test step.
 
         received:
             The received response.

scripts/py_matter_yamltests/matter_yamltests/parser.py

@@ -180,6 +180,7 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_
         self.attribute = _value_or_none(test, 'attribute')
         self.event = _value_or_none(test, 'event')
         self.endpoint = _value_or_config(test, 'endpoint', config)
+        self.pics = _value_or_none(test, 'PICS')
         self.is_pics_enabled = pics_checker.check(_value_or_none(test, 'PICS'))
 
         self.identity = _value_or_none(test, 'identity')
@@ -565,6 +566,10 @@ def wait_for(self):
     def event_number(self):
         return self._test.event_number
 
+    @property
+    def pics(self):
+        return self._test.pics
+
     def post_process_response(self, received_responses):
         result = PostProcessResponseResult()
 
@@ -955,11 +960,12 @@ class TestParserConfig:
 class TestParser:
     def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserConfig()):
         yaml_loader = YamlLoader()
-        name, pics, config, tests = yaml_loader.load(test_file)
+        filename, name, pics, config, tests = yaml_loader.load(test_file)
 
         self.__apply_config_override(config, parser_config.config_override)
         self.__apply_legacy_config(config)
 
+        self.filename = filename
         self.name = name
         self.PICS = pics
         self.tests = YamlTests(

scripts/py_matter_yamltests/matter_yamltests/runner.py

@@ -157,12 +157,12 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig):
             await self.start()
 
             hooks = config.hooks
-            hooks.test_start(parser.name, parser.tests.count)
+            hooks.test_start(parser.filename, parser.name, parser.tests.count)
 
             test_duration = 0
             for idx, request in enumerate(parser.tests):
                 if not request.is_pics_enabled:
-                    hooks.step_skipped(request.label)
+                    hooks.step_skipped(request.label, request.pics)
                     continue
                 elif not config.adapter:
                     hooks.step_start(request.label)
@@ -185,9 +185,9 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig):
 
                 if logger.is_failure():
                     hooks.step_failure(logger, logs, duration,
-                                       request.responses, responses)
+                                       request, responses)
                 else:
-                    hooks.step_success(logger, logs, duration)
+                    hooks.step_success(logger, logs, duration, request)
 
                 if logger.is_failure() and config.options.stop_on_error:
                     status = False

scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py

@@ -25,19 +25,23 @@
 except:
     from yaml import SafeLoader
 
+import os
+
 import yaml
 
 
 class YamlLoader:
     """This class loads a file from the disk and validates that the content is a well formed yaml test."""
 
     def load(self, yaml_file: str) -> tuple[str, Union[list, str], dict, list]:
+        filename = ''
         name = ''
         pics = None
         config = {}
         tests = []
 
         if yaml_file:
+            filename = os.path.splitext(os.path.basename(yaml_file))[0]
             with open(yaml_file) as f:
                 loader = SafeLoader
                 add_yaml_support_for_scientific_notation_without_dot(loader)
@@ -50,7 +54,7 @@ def load(self, yaml_file: str) -> tuple[str, Union[list, str], dict, list]:
                 config = content.get('config', {})
                 tests = content.get('tests', [])
 
-        return (name, pics, config, tests)
+        return (filename, name, pics, config, tests)
 
     def __check_content(self, content):
         schema = {

scripts/py_matter_yamltests/test_yaml_loader.py

@@ -66,7 +66,8 @@ def test_missing_file(self):
 
         content = None
 
-        name, pics, config, tests = load(content)
+        filename, name, pics, config, tests = load(content)
+        self.assertEqual(filename, '')
         self.assertEqual(name, '')
         self.assertEqual(pics, None)
         self.assertEqual(config, {})
@@ -77,7 +78,8 @@ def test_empty_file(self):
 
         content = ''
 
-        name, pics, config, tests = load(content)
+        filename, name, pics, config, tests = load(content)
+        self.assertEqual(name, '')
         self.assertEqual(name, '')
         self.assertEqual(pics, None)
         self.assertEqual(config, {})
@@ -99,7 +101,7 @@ def test_key_name(self):
         name: Test Name
         '''
 
-        name, _, _, _ = load(content)
+        _, name, _, _, _ = load(content)
         self.assertEqual(name, 'Test Name')
 
     def test_key_name_wrong_values(self):
@@ -117,7 +119,7 @@ def test_key_pics_string(self):
         PICS: OO.S
         '''
 
-        _, pics, _, _ = load(content)
+        _, _, pics, _, _ = load(content)
         self.assertEqual(pics, 'OO.S')
 
     def test_key_pics_list(self):
@@ -129,7 +131,7 @@ def test_key_pics_list(self):
             - OO.C
         '''
 
-        _, pics, _, _ = load(content)
+        _, _, pics, _, _ = load(content)
         self.assertEqual(pics, ['OO.S', 'OO.C'])
 
     def test_key_pics_wrong_values(self):
@@ -149,7 +151,7 @@ def test_key_config(self):
             name2: value2
         '''
 
-        _, _, config, _ = load(content)
+        _, _, _, config, _ = load(content)
         self.assertEqual(config, {'name': 'value', 'name2': 'value2'})
 
     def test_key_config_wrong_values(self):
@@ -169,7 +171,7 @@ def test_key_tests(self):
             - label: Test2
         '''
 
-        _, _, _, tests = load(content)
+        _, _, _, _, tests = load(content)
         self.assertEqual(tests, [{'label': 'Test1'}, {'label': 'Test2'}])
 
     def test_key_tests_wrong_values(self):
@@ -202,7 +204,7 @@ def test_key_tests_step_bool_keys(self):
 
         wrong_values = self._get_wrong_values([bool], spaces=6)
         for key in keys:
-            _, _, _, tests = load(content.format(key=key, value=True))
+            _, _, _, _, tests = load(content.format(key=key, value=True))
             self.assertEqual(tests, [{key: True}])
 
             for value in wrong_values:
@@ -232,7 +234,7 @@ def test_key_tests_step_str_keys(self):
 
         wrong_values = self._get_wrong_values([str], spaces=6)
         for key in keys:
-            _, _, _, tests = load(content.format(key=key, value='a string'))
+            _, _, _, _, tests = load(content.format(key=key, value='a string'))
             self.assertEqual(tests, [{key: 'a string'}])
 
             for value in wrong_values:
@@ -256,7 +258,7 @@ def test_key_tests_step_int_keys(self):
 
         wrong_values = self._get_wrong_values([int], spaces=6)
         for key in keys:
-            _, _, _, tests = load(content.format(key=key, value=1))
+            _, _, _, _, tests = load(content.format(key=key, value=1))
             self.assertEqual(tests, [{key: 1}])
 
             for value in wrong_values:
@@ -276,7 +278,8 @@ def test_key_tests_step_dict_keys(self):
                        '      value: True\n')
         wrong_values = self._get_wrong_values([dict], spaces=6)
         for key in keys:
-            _, _, _, tests = load(content.format(key=key, value=valid_value))
+            _, _, _, _, tests = load(
+                content.format(key=key, value=valid_value))
             self.assertEqual(tests, [{key: {'value': True}}])
 
             for value in wrong_values:
@@ -291,12 +294,12 @@ def test_key_tests_step_response_key(self):
 
         value = ('\n'
                  '      value: True\n')
-        _, _, _, tests = load(content.format(value=value))
+        _, _, _, _, tests = load(content.format(value=value))
         self.assertEqual(tests, [{'response': {'value': True}}])
 
         value = ('\n'
                  '      - value: True\n')
-        _, _, _, tests = load(content.format(value=value))
+        _, _, _, _, tests = load(content.format(value=value))
         self.assertEqual(tests, [{'response': [{'value': True}]}])
 
         wrong_values = self._get_wrong_values([dict, list], spaces=6)
@@ -310,10 +313,10 @@ def test_key_tests_step_event_number_key(self):
         content = ('tests:\n'
                    '  - eventNumber: {value}')
 
-        _, _, _, tests = load(content.format(value=1))
+        _, _, _, _, tests = load(content.format(value=1))
         self.assertEqual(tests, [{'eventNumber': 1}])
 
-        _, _, _, tests = load(content.format(value='TestKey'))
+        _, _, _, _, tests = load(content.format(value='TestKey'))
         self.assertEqual(tests, [{'eventNumber': 'TestKey'}])
 
         wrong_values = self._get_wrong_values([str, int], spaces=6)
@@ -328,7 +331,7 @@ def test_key_tests_step_verification_key(self):
                    '  - verification: {value}\n'
                    '    disabled: true')
 
-        _, _, _, tests = load(content.format(value='Test Sentence'))
+        _, _, _, _, tests = load(content.format(value='Test Sentence'))
         self.assertEqual(
             tests, [{'verification': 'Test Sentence', 'disabled': True}])
 
@@ -392,7 +395,7 @@ def test_key_tests_step_rule_step_with_verification_should_be_disabled_or_intera
                 disabled: true
         '''
 
-        _, _, _, tests = load(content)
+        _, _, _, _, tests = load(content)
         self.assertEqual(tests, [
                          {'label': 'A Test Name', 'verification': 'A verification sentence', 'disabled': True}])
 
@@ -412,7 +415,7 @@ def test_key_tests_step_rule_step_with_verification_should_be_disabled_or_intera
                 command: UserPrompt
         '''
 
-        _, _, _, tests = load(content)
+        _, _, _, _, tests = load(content)
         self.assertEqual(tests, [
                          {'label': 'A Test Name', 'verification': 'A verification sentence', 'command': 'UserPrompt'}])
 
@@ -427,7 +430,7 @@ def test_key_tests_step_response_key_values_key(self):
                    '  - response:\n'
                    '      values: {value}')
 
-        _, _, _, tests = load(content.format(value=[]))
+        _, _, _, _, tests = load(content.format(value=[]))
         self.assertEqual(tests, [{'response': {'values': []}}])
 
         wrong_values = self._get_wrong_values([list], spaces=8)
@@ -442,7 +445,7 @@ def test_key_tests_step_response_key_error_key(self):
                    '  - response:\n'
                    '      error: {value}')
 
-        _, _, _, tests = load(content.format(value='AnError'))
+        _, _, _, _, tests = load(content.format(value='AnError'))
         self.assertEqual(tests, [{'response': {'error': 'AnError'}}])
 
         wrong_values = self._get_wrong_values([str], spaces=8)
@@ -457,7 +460,7 @@ def test_key_tests_step_response_key_cluster_error_key(self):
                    '  - response:\n'
                    '      clusterError: {value}')
 
-        _, _, _, tests = load(content.format(value=1))
+        _, _, _, _, tests = load(content.format(value=1))
         self.assertEqual(tests, [{'response': {'clusterError': 1}}])
 
         wrong_values = self._get_wrong_values([int], spaces=8)
@@ -472,7 +475,7 @@ def test_key_tests_step_response_key_constraints_key(self):
                    '  - response:\n'
                    '      constraints: {value}')
 
-        _, _, _, tests = load(content.format(value={}))
+        _, _, _, _, tests = load(content.format(value={}))
         self.assertEqual(tests, [{'response': {'constraints': {}}}])
 
         wrong_values = self._get_wrong_values([dict], spaces=8)
@@ -487,7 +490,7 @@ def test_key_tests_step_response_key_save_as_key(self):
                    '  - response:\n'
                    '      saveAs: {value}')
 
-        _, _, _, tests = load(content.format(value='AKey'))
+        _, _, _, _, tests = load(content.format(value='AKey'))
         self.assertEqual(tests, [{'response': {'saveAs': 'AKey'}}])
 
         wrong_values = self._get_wrong_values([str], spaces=8)

scripts/tests/yaml/chiptool.py

@@ -85,6 +85,7 @@ def chiptool_runner_options(f):
 
 
 CONTEXT_SETTINGS['ignore_unknown_options'] = True
+CONTEXT_SETTINGS['default_map']['chiptool']['use_test_harness_log_format'] = True
 
 
 @click.command(context_settings=CONTEXT_SETTINGS)