Python packaging idl and yamltests (#24140)

* Initial attempt at modularizing python packages in scripts

* Put packages into separate directories with stutter

* hopefully fix codegen path to idl

* Fix issue with building yamltests_distribution

* Hopefully fix CI

* Hopefully fix CI

* Hopefully fix CI

* Restyle

* Hopefully fix CI

* Restyle

* Remove directory repitition and prepend matter to py package names

* Restyle

* Restyle

.github/workflows/lint.yml

@@ -98,13 +98,13 @@ jobs:
                      if [ "$idl_file" = './examples/window-app/common/window-app.matter' ]; then continue; fi
 
                      # Test files are intentionally small and not spec-compilant, just parse-compliant
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/cluster_struct_attribute.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/global_struct_attribute.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/optional_argument.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/several_clusters.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/simple_attribute.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/large_lighting_app.matter" ]; then continue; fi
-                     if [ "$idl_file" = "./scripts/idl/tests/inputs/large_all_clusters_app.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/cluster_struct_attribute.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/global_struct_attribute.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/optional_argument.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/several_clusters.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/simple_attribute.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/large_lighting_app.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/py_matter_idl/matter_idl/tests/inputs/large_all_clusters_app.matter" ]; then continue; fi
 
                      ./scripts/run_in_build_env.sh "./scripts/idl_lint.py --log-level warn $idl_file" >/dev/null || exit 1
                  done

.github/workflows/tests.yaml

@@ -105,7 +105,7 @@ jobs:
               #       things.
               run: |
                   ./scripts/run_in_build_env.sh \
-                    "./scripts/idl/xml_parser.py \
+                    "./scripts/py_matter_idl/matter_idl/xml_parser.py \
                       --no-print \
                       --log-level info \
                       src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \

.restyled.yaml

@@ -69,7 +69,7 @@ exclude:
     - "third_party/android_deps/gradlew" # gradle wrapper generated file
     - "src/controller/python/chip/clusters/Objects.py" # generated file, no point to restyle
     - "src/controller/python/chip/clusters/CHIPClusters.py" # generated file, no point to restyle
-    - "scripts/idl/tests/outputs/**/*" # Matches generated output 1:1
+    - "scripts/py_matter_idl/matter_idl/tests/outputs/**/*" # Matches generated output 1:1
     - "examples/chef/sample_app_util/test_files/*.yaml"
     - "examples/chef/zzz_generated/**/*"
     - "examples/platform/nxp/k32w/k32w0/scripts/demo_generated_certs/**/*"

BUILD.gn

@@ -120,7 +120,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
   }
 
   pw_python_pip_install("pip_install_matter_packages") {
-    packages = [ "//examples/common/pigweed/rpc_console:chip_rpc_distribution" ]
+    packages = [
+      "//examples/common/pigweed/rpc_console:chip_rpc_distribution",
+      "//scripts:yamltests_distribution",
+    ]
   }
 
   # Python packages installed during bootstrap.
@@ -260,7 +263,8 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         "//:fake_platform_tests",
         "//examples/chef:chef.tests",
         "//scripts/build:build_examples.tests",
-        "//scripts/idl:idl.tests",
+        "//scripts/py_matter_idl:matter_idl.tests",
+        "//scripts/py_matter_yamltests:matter_yamltests.tests",
         "//src:tests_run",
       ]
     }

build/chip/chip_codegen.gni

@@ -17,7 +17,7 @@ import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 
 import("$dir_pw_build/python.gni")
-import("${chip_root}/scripts/idl/files.gni")
+import("${chip_root}/scripts/py_matter_idl/files.gni")
 
 declare_args() {
   # Location where code has been pre-generated

docs/code_generation.md

@@ -84,9 +84,10 @@ specific codegen.
 ### `*.matter` parsing and codegen
 
 `*.matter` files are both human and machine readable. Code that can process
-these files is available at `scripts/idl` and `scripts/codegen.py`. You can read
-the [scripts/idl/README.md](../scripts/idl/README.md) for details of how things
-work.
+these files is available at `scripts/py_matter_idl` and `scripts/codegen.py`.
+You can read the
+[scripts/py_matter_idl/matter_idl/README.md](../scripts/py_matter_idl/matter_idl/README.md)
+for details of how things work.
 
 `scripts/codegen.py` can generate various outputs based on an input `*.matter`
 file.
@@ -198,7 +199,7 @@ Code pre-generation can be used:
     generation at build time or to save the code generation time at the expense
     of running code generation for every possible zap/generation type
 -   To check changes in generated code across versions, beyond the comparisons
-    of golden image tests in `scripts/idl/tests`
+    of golden image tests in `scripts/py_matter_idl/matter_idl/tests`
 
 The script to trigger code pre-generation is `scripts/code_pregenerate.py` and
 requires the pre-generation output directory as an argument

scripts/BUILD.gn

@@ -12,6 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import("//build_overrides/build.gni")
+import("//build_overrides/chip.gni")
+
 import("//build_overrides/pigweed.gni")
+import("$dir_pw_build/python_dist.gni")
 
-import("$dir_pw_build/python.gni")
+# This target creates a single Python package and wheel for yamltests. It will
+# merge all Python dependencies Matter. The output is located in:
+#   out/obj/yamltests_distribution/  <- source files here
+#   out/obj/yamltests_distribution._build_wheel/yamltests-0.0.1-py3-none-any.whl
+pw_python_distribution("yamltests_distribution") {
+  packages = [ "${chip_root}/scripts/py_matter_yamltests:matter_yamltests" ]
+  generate_setup_cfg = {
+    common_config_file = "common_setup.cfg"
+    include_default_pyproject_file = true
+    append_date_to_version = true
+  }
+}

scripts/codegen.py

@@ -25,14 +25,14 @@
     _has_coloredlogs = False
 
 try:
-    from idl.matter_idl_parser import CreateParser
+    from matter_idl.matter_idl_parser import CreateParser
 except:
     import os
-    sys.path.append(os.path.abspath(os.path.dirname(__file__)))
-    from idl.matter_idl_parser import CreateParser
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'py_matter_idl')))
+    from matter_idl.matter_idl_parser import CreateParser
 
-from idl.generators import FileSystemGeneratorStorage, GeneratorStorage
-from idl.generators.registry import CodeGenerator, GENERATORS
+from matter_idl.generators import FileSystemGeneratorStorage, GeneratorStorage
+from matter_idl.generators.registry import CodeGenerator, GENERATORS
 
 
 class ListGeneratedFilesStorage(GeneratorStorage):

scripts/common_setup.cfg

@@ -0,0 +1,19 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[metadata]
+name = matter_yamltests
+version = 0.0.1
+
+[options]
+zip_safe = False

scripts/idl_lint.py

@@ -23,12 +23,12 @@
 from typing import List, Optional
 
 try:
-    from idl import matter_idl_parser
+    from matter_idl import matter_idl_parser
 except:
-    sys.path.append(os.path.abspath(os.path.dirname(__file__)))
-    from idl import matter_idl_parser
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'py_matter_idl')))
+    from matter_idl import matter_idl_parser
 
-import idl.lint
+import matter_idl.lint
 
 # Supported log levels, mapping string values required for argument
 # parsing into logging constants
@@ -64,7 +64,7 @@ def main(log_level, rules, idl_path):
 
     lint_rules = []
     logging.info("Loading rules from %s" % rules)
-    lint_rules.extend(idl.lint.CreateParser(rules).parse())
+    lint_rules.extend(matter_idl.lint.CreateParser(rules).parse())
 
     logging.info("Parsing idl from %s" % idl_path)
     idl_tree = matter_idl_parser.CreateParser().parse(open(idl_path, "rt").read(), file_name=idl_path)

scripts/py_matter_idl/BUILD.gn

@@ -0,0 +1,83 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build_overrides/build.gni")
+import("//build_overrides/chip.gni")
+
+import("//build_overrides/pigweed.gni")
+import("$dir_pw_build/python.gni")
+
+import("${chip_root}/scripts/py_matter_idl/files.gni")
+
+pw_python_package("matter_idl") {
+  setup = [
+    "setup.py",
+    "setup.cfg",
+    "pyproject.toml",
+  ]
+  inputs = matter_idl_generator_templates
+  inputs += [
+    # Dependency grammar
+    "matter_idl/matter_grammar.lark",
+
+    # Unit test data
+    "matter_idl/tests/available_tests.yaml",
+    "matter_idl/tests/inputs/cluster_struct_attribute.matter",
+    "matter_idl/tests/inputs/global_struct_attribute.matter",
+    "matter_idl/tests/inputs/optional_argument.matter",
+    "matter_idl/tests/inputs/several_clusters.matter",
+    "matter_idl/tests/inputs/simple_attribute.matter",
+    "matter_idl/tests/outputs/cluster_struct_attribute/bridge/BridgeClustersImpl.h",
+    "matter_idl/tests/outputs/cluster_struct_attribute/bridge/BridgeGlobalStructs.h",
+    "matter_idl/tests/outputs/cluster_struct_attribute/bridge/DemoClusterServer.h",
+    "matter_idl/tests/outputs/cluster_struct_attribute/jni/DemoClusterClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/cluster_struct_attribute/jni/DemoClusterClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/global_struct_attribute/bridge/BridgeClustersImpl.h",
+    "matter_idl/tests/outputs/global_struct_attribute/bridge/BridgeGlobalStructs.h",
+    "matter_idl/tests/outputs/global_struct_attribute/bridge/DemoClusterServer.h",
+    "matter_idl/tests/outputs/global_struct_attribute/jni/DemoClusterClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/global_struct_attribute/jni/DemoClusterClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/optional_argument/jni/MyClusterClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/optional_argument/jni/MyClusterClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/bridge/BridgeClustersImpl.h",
+    "matter_idl/tests/outputs/several_clusters/bridge/BridgeGlobalStructs.h",
+    "matter_idl/tests/outputs/several_clusters/bridge/FirstServer.h",
+    "matter_idl/tests/outputs/several_clusters/bridge/SecondServer.h",
+    "matter_idl/tests/outputs/several_clusters/bridge/Third.h",
+    "matter_idl/tests/outputs/several_clusters/bridge/ThirdServer.h",
+    "matter_idl/tests/outputs/several_clusters/jni/FirstClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/jni/SecondClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/jni/ThirdClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/jni/FirstClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/jni/SecondClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/several_clusters/jni/ThirdClient-InvokeSubscribeImpl.cpp",
+    "matter_idl/tests/outputs/simple_attribute/bridge/BridgeClustersImpl.h",
+    "matter_idl/tests/outputs/simple_attribute/bridge/BridgeGlobalStructs.h",
+    "matter_idl/tests/outputs/simple_attribute/bridge/MyClusterServer.h",
+    "matter_idl/tests/outputs/simple_attribute/jni/MyClusterClient-ReadImpl.cpp",
+    "matter_idl/tests/outputs/simple_attribute/jni/MyClusterClient-InvokeSubscribeImpl.cpp",
+  ]
+
+  sources = matter_idl_generator_sources
+
+  tests = [
+    "matter_idl/test_matter_idl_parser.py",
+    "matter_idl/test_generators.py",
+    "matter_idl/test_xml_parser.py",
+  ]
+
+  # TODO: at a future time consider enabling all (* or missing) here to get
+  #       pylint checking these files
+  static_analysis = []
+}