Add a mandatory fence to CI tests config and improve Python testing d…

…ocs (#34319) * Update docs and introduce fences for CI test metadata * Add fences to all tests * Update docs/testing/python.md Co-authored-by: C Freeman <cecille@google.com> * Restyled by prettier-markdown * Restyled by autopep8 * Restyled by isort --------- Co-authored-by: C Freeman <cecille@google.com> Co-authored-by: Restyled.io <commits@restyled.io>
project-chip · Sep 6, 2024 · 6baf344 · 6baf344
1 parent 534b53b
commit 6baf344
Show file tree

Hide file tree

Showing 85 changed files with 651 additions and 185 deletions.
diff --git a/docs/testing/python.md b/docs/testing/python.md
diff --git a/scripts/tests/py/metadata.py b/scripts/tests/py/metadata.py
@@ -14,12 +14,18 @@
 # limitations under the License.
 
 import re
+import sys
 from dataclasses import dataclass
 from typing import Any, Dict, List
 
 import yaml
 
 
+def bool_from_str(value: str) -> bool:
+    """Convert True/true/False/false strings to bool."""
+    return value.strip().lower() == "true"
+
+
 @dataclass
 class Metadata:
     py_script_path: str
@@ -58,16 +64,59 @@ def copy_from_dict(self, attr_dict: Dict[str, Any]) -> None:
             self.py_script_path = attr_dict["py_script_path"]
 
         if "factoryreset" in attr_dict:
-            self.factoryreset = bool(attr_dict["factoryreset"])
+            self.factoryreset = bool_from_str(attr_dict["factoryreset"])
 
         if "factoryreset_app_only" in attr_dict:
-            self.factoryreset_app_only = bool(attr_dict["factoryreset_app_only"])
+            self.factoryreset_app_only = bool_from_str(attr_dict["factoryreset_app_only"])
 
         if "script_gdb" in attr_dict:
-            self.script_gdb = bool(attr_dict["script_gdb"])
+            self.script_gdb = bool_from_str(attr_dict["script_gdb"])
 
         if "quiet" in attr_dict:
-            self.quiet = bool(attr_dict["quiet"])
+            self.quiet = bool_from_str(attr_dict["quiet"])
+
+
+def extract_runs_arg_lines(py_script_path: str) -> Dict[str, Dict[str, str]]:
+    """Extract the run arguments from the CI test arguments blocks."""
+
+    found_ci_args_section = False
+    done_ci_args_section = False
+
+    runs_def_ptrn = re.compile(r'^\s*#\s*test-runner-runs:\s*(?P<run_id>.*)$')
+    arg_def_ptrn = re.compile(
+        r'^\s*#\s*test-runner-run/(?P<run_id>[a-zA-Z0-9_]+)/(?P<arg_name>[a-zA-Z0-9_\-]+):\s*(?P<arg_val>.*)$')
+
+    runs_arg_lines: Dict[str, Dict[str, str]] = {}
+
+    with open(py_script_path, 'r', encoding='utf8') as py_script:
+        for line_idx, line in enumerate(py_script.readlines()):
+            line = line.strip()
+            line_num = line_idx + 1
+
+            # Detect the single CI args section, to skip the lines otherwise.
+            if not done_ci_args_section and line.startswith("# === BEGIN CI TEST ARGUMENTS ==="):
+                found_ci_args_section = True
+            elif found_ci_args_section and line.startswith("# === END CI TEST ARGUMENTS ==="):
+                done_ci_args_section = True
+                found_ci_args_section = False
+
+            runs_match = runs_def_ptrn.match(line)
+            args_match = arg_def_ptrn.match(line)
+
+            if not found_ci_args_section and (runs_match or args_match):
+                print(f"WARNING: {py_script_path}:{line_num}: Found CI args outside of CI TEST ARGUMENTS block!", file=sys.stderr)
+                continue
+
+            if runs_match:
+                for run in runs_match.group("run_id").strip().split():
+                    runs_arg_lines[run] = {}
+                    runs_arg_lines[run]['run'] = run
+                    runs_arg_lines[run]['py_script_path'] = py_script_path
+
+            elif args_match:
+                runs_arg_lines[args_match.group("run_id")][args_match.group("arg_name")] = args_match.group("arg_val")
+
+    return runs_arg_lines
 
 
 class MetadataReader:
@@ -126,33 +175,15 @@ def parse_script(self, py_script_path: str) -> List[Metadata]:
          the run arguments associated with a particular run defined in
          the script file.
         """
-
-        runs_def_ptrn = re.compile(r'^\s*#\s*test-runner-runs:\s*(.*)$')
-        arg_def_ptrn = re.compile(r'^\s*#\s*test-runner-run/([a-zA-Z0-9_]+)/([a-zA-Z0-9_\-]+):\s*(.*)$')
-
-        runs_arg_lines: Dict[str, Dict[str, str]] = {}
         runs_metadata: List[Metadata] = []
-
-        with open(py_script_path, 'r', encoding='utf8') as py_script:
-            for line in py_script.readlines():
-                runs_match = runs_def_ptrn.match(line.strip())
-                args_match = arg_def_ptrn.match(line.strip())
-
-                if runs_match:
-                    for run in runs_match.group(1).strip().split():
-                        runs_arg_lines[run] = {}
-                        runs_arg_lines[run]['run'] = run
-                        runs_arg_lines[run]['py_script_path'] = py_script_path
-
-                elif args_match:
-                    runs_arg_lines[args_match.group(1)][args_match.group(2)] = args_match.group(3)
+        runs_arg_lines = extract_runs_arg_lines(py_script_path)
 
         for run, attr in runs_arg_lines.items():
             self.__resolve_env_vals__(attr)
 
             metadata = Metadata(
                 py_script_path=attr.get("py_script_path", ""),
-                run=attr.get("run", ""),
+                run=run,
                 app=attr.get("app", ""),
                 app_args=attr.get("app_args", ""),
                 script_args=attr.get("script_args", ""),

diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py
@@ -21,13 +21,17 @@
 
 class TestMetadataReader(unittest.TestCase):
 
-    test_file_content = ''' 
-    # test-runner-runs: run1 
+    test_file_content = '''
+    # === BEGIN CI TEST ARGUMENTS ===
+    # test-runner-runs: run1
     # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
     # test-runner-run/run1/app-args: --discriminator 1234 --trace-to json:${TRACE_APP}.json
     # test-runner-run/run1/script-args: --commissioning-method on-network --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
     # test-runner-run/run1/factoryreset: True
     # test-runner-run/run1/quiet: True
+    # === END CI TEST ARGUMENTS ===
+
+    # test-runner-run/run1/quiet: False
     '''
 
     env_file_content = '''

diff --git a/src/controller/python/test/test_scripts/mobile-device-test.py b/src/controller/python/test/test_scripts/mobile-device-test.py
@@ -19,12 +19,17 @@
 
 # Commissioning test.
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import asyncio
 import os

diff --git a/src/python_testing/TC_ACE_1_2.py b/src/python_testing/TC_ACE_1_2.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 import queue

diff --git a/src/python_testing/TC_ACE_1_3.py b/src/python_testing/TC_ACE_1_3.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 

diff --git a/src/python_testing/TC_ACE_1_4.py b/src/python_testing/TC_ACE_1_4.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import sys
 

diff --git a/src/python_testing/TC_ACE_1_5.py b/src/python_testing/TC_ACE_1_5.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 

diff --git a/src/python_testing/TC_AccessChecker.py b/src/python_testing/TC_AccessChecker.py
@@ -1,9 +1,14 @@
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 from copy import deepcopy

diff --git a/src/python_testing/TC_CGEN_2_4.py b/src/python_testing/TC_CGEN_2_4.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 import random

diff --git a/src/python_testing/TC_DA_1_2.py b/src/python_testing/TC_DA_1_2.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import os
 import random

diff --git a/src/python_testing/TC_DA_1_5.py b/src/python_testing/TC_DA_1_5.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import random
 

diff --git a/src/python_testing/TC_DA_1_7.py b/src/python_testing/TC_DA_1_7.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --bool-arg allow_sdk_dac:true --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import logging
 from glob import glob

diff --git a/src/python_testing/TC_DGGEN_2_4.py b/src/python_testing/TC_DGGEN_2_4.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 import asyncio
 import logging

diff --git a/src/python_testing/TC_DRLK_2_12.py b/src/python_testing/TC_DRLK_2_12.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${CHIP_LOCK_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 from drlk_2_x_common import DRLK_COMMON
 from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main

diff --git a/src/python_testing/TC_DRLK_2_2.py b/src/python_testing/TC_DRLK_2_2.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${CHIP_LOCK_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 from drlk_2_x_common import DRLK_COMMON
 from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main

diff --git a/src/python_testing/TC_DRLK_2_3.py b/src/python_testing/TC_DRLK_2_3.py
@@ -15,12 +15,17 @@
 #    limitations under the License.
 #
 
+# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
+# for details about the block below.
+#
+# === BEGIN CI TEST ARGUMENTS ===
 # test-runner-runs: run1
 # test-runner-run/run1/app: ${CHIP_LOCK_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# === END CI TEST ARGUMENTS ===
 
 from drlk_2_x_common import DRLK_COMMON
 from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main