Add a linter to validate that our configuration for codegen actually …

…implements all requirements (#19074)

* Add support for a linter program for matter files:

- supports a simplified parser language for "rules"
- supports loading existing XML files that define cluster data and
  required attributes

* code review

* Updated error message

* Updated error message again - better description

scripts/idl/lint/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .lint_rules_parser import CreateParser

scripts/idl/lint/lint_rules_grammar.lark

@@ -0,0 +1,32 @@
+start: instruction*
+
+instruction: load_xml|all_endpoint_rule|specific_endpoint_rule
+
+load_xml: "load" ESCAPED_STRING ";"
+
+all_endpoint_rule: "all" "endpoints" "{" required_global_attribute* "}"
+
+specific_endpoint_rule: "endpoint" integer "{" required_server_cluster* "}"
+
+required_global_attribute: "require" "global" "attribute" id "=" integer ";"
+
+required_server_cluster: "require" "server" "cluster" id ";"
+
+integer: positive_integer | negative_integer
+
+positive_integer: POSITIVE_INTEGER | HEX_INTEGER
+negative_integer: "-" positive_integer
+
+id: ID
+
+POSITIVE_INTEGER: /\d+/ 
+HEX_INTEGER: /0x[A-Fa-f0-9]+/ 
+ID: /[a-zA-Z_][a-zA-Z0-9_]*/
+
+%import common.ESCAPED_STRING
+%import common.WS
+%import common.C_COMMENT
+%import common.CPP_COMMENT
+%ignore WS
+%ignore C_COMMENT
+%ignore CPP_COMMENT

scripts/idl/lint/lint_rules_parser.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import xml.etree.ElementTree
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Mapping
+from lark import Lark
+from lark.visitors import Transformer, v_args, Discard
+import stringcase
+import traceback
+
+try:
+    from .types import RequiredAttributesRule, AttributeRequirement, ClusterRequirement
+except:
+    import sys
+
+    sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", ".."))
+    from idl.lint.types import RequiredAttributesRule, AttributeRequirement, ClusterRequirement
+
+
+def parseNumberString(n):
+    if n.startswith('0x'):
+        return int(n[2:], 16)
+    else:
+        return int(n)
+
+
+@dataclass
+class RequiredAttribute:
+    name: str
+    code: int
+
+
+@dataclass
+class DecodedCluster:
+    name: str
+    code: int
+    required_attributes: List[RequiredAttribute]
+
+
+def DecodeClusterFromXml(element: xml.etree.ElementTree.Element):
+    if element.tag != 'cluster':
+        logging.error("Not a cluster element: %r" % element)
+        return None
+
+    # cluster elements contain among other children
+    #  - name (general name for this cluster)
+    #  - code (unique identifier, may be hex or numeric)
+    #  - attribute with side, code and optional attributes
+
+    try:
+        name = element.find('name').text.replace(' ', '')
+        required_attributes = []
+
+        for attr in element.findall('attribute'):
+            if attr.attrib['side'] != 'server':
+                continue
+
+            if 'optional' in attr.attrib and attr.attrib['optional'] == 'true':
+                continue
+
+            required_attributes.append(
+                RequiredAttribute(
+                    name=attr.text,
+                    code=parseNumberString(attr.attrib['code'])
+                ))
+
+        return DecodedCluster(
+            name=name,
+            code=parseNumberString(element.find('code').text),
+            required_attributes=required_attributes,
+        )
+    except Exception as e:
+        logging.exception("Failed to decode cluster %r" % element)
+        return None
+
+
+def ClustersInXmlFile(path: str):
+    logging.info("Loading XML from %s" % path)
+
+    # root is expected to be just a "configurator" object
+    configurator = xml.etree.ElementTree.parse(path).getroot()
+    for child in configurator:
+        if child.tag != 'cluster':
+            continue
+        yield child
+
+
+class LintRulesContext:
+    """Represents a context for loadint lint rules.
+
+    Handles:
+       - loading referenced files (matter xml definitions)
+       - adding linter rules as data is parsed
+       - Looking up identifiers for various rules
+    """
+
+    def __init__(self):
+        self._linter_rule = RequiredAttributesRule("Rules file")
+
+        # Map cluster names to the underlying code
+        self._cluster_codes: Mapping[str, int] = {}
+
+    def GetLinterRules(self):
+        return [self._linter_rule]
+
+    def RequireAttribute(self, r: AttributeRequirement):
+        self._linter_rule.RequireAttribute(r)
+
+    def RequireClusterInEndpoint(self, name: str, code: int):
+        """Mark that a specific cluster is always required in the given endpoint
+        """
+        if name not in self._cluster_codes:
+            logging.error("UNKNOWN cluster name %s" % name)
+            logging.error("Known names: %s" % (",".join(self._cluster_codes.keys()), ))
+            return
+
+        self._linter_rule.RequireClusterInEndpoint(ClusterRequirement(
+            endpoint_id=code,
+            cluster_id=self._cluster_codes[name],
+            cluster_name=name,
+        ))
+
+    def LoadXml(self, path: str):
+        """Load XML data from the given path and add it to 
+           internal processing. Adds attribute requirement rules
+           as needed.
+        """
+        for cluster in ClustersInXmlFile(path):
+            decoded = DecodeClusterFromXml(cluster)
+
+            if not decoded:
+                continue
+
+            self._cluster_codes[decoded.name] = decoded.code
+
+            for attr in decoded.required_attributes:
+                self._linter_rule.RequireAttribute(AttributeRequirement(
+                    code=attr.code, name=attr.name, filter_cluster=decoded.code))
+
+            # TODO: add cluster ID to internal registry
+
+
+class LintRulesTransformer(Transformer):
+    """
+    A transformer capable to transform data parsed by Lark according to 
+    lint_rules_grammar.lark.
+    """
+
+    def __init__(self, file_name: str):
+        self.context = LintRulesContext()
+        self.file_name = file_name
+
+    def positive_integer(self, tokens):
+        """Numbers in the grammar are integers or hex numbers.
+        """
+        if len(tokens) != 1:
+            raise Error("Unexpected argument counts")
+
+        return parseNumberString(tokens[0].value)
+
+    @v_args(inline=True)
+    def negative_integer(self, value):
+        return -value
+
+    @v_args(inline=True)
+    def integer(self, value):
+        return value
+
+    def id(self, tokens):
+        """An id is a string containing an identifier
+        """
+        if len(tokens) != 1:
+            raise Error("Unexpected argument counts")
+        return tokens[0].value
+
+    def ESCAPED_STRING(self, s):
+        # handle escapes, skip the start and end quotes
+        return s.value[1:-1].encode('utf-8').decode('unicode-escape')
+
+    def start(self, instructions):
+        # At this point processing is considered done, return all
+        # linter rules that were found
+        return self.context.GetLinterRules()
+
+    def instruction(self, instruction):
+        return Discard
+
+    def all_endpoint_rule(self, attributes):
+        for attribute in attributes:
+            self.context.RequireAttribute(attribute)
+
+        return Discard
+
+    @v_args(inline=True)
+    def load_xml(self, path):
+        if not os.path.isabs(path):
+            path = os.path.abspath(os.path.join(os.path.dirname(self.file_name), path))
+
+        self.context.LoadXml(path)
+
+    @v_args(inline=True)
+    def required_global_attribute(self, name, code):
+        return AttributeRequirement(code=code, name=name)
+
+    @v_args(inline=True)
+    def specific_endpoint_rule(self, code, *names):
+        for name in names:
+            self.context.RequireClusterInEndpoint(name, code)
+        return Discard
+
+    @v_args(inline=True)
+    def required_server_cluster(self, id):
+        return id
+
+
+class Parser:
+    def __init__(self, parser, file_name: str):
+        self.parser = parser
+        self.file_name = file_name
+
+    def parse(self):
+        data = LintRulesTransformer(self.file_name).transform(self.parser.parse(open(self.file_name, "rt").read()))
+        return data
+
+
+def CreateParser(file_name: str):
+    """
+    Generates a parser that will process a ".matter" file into a IDL
+    """
+    return Parser(Lark.open('lint_rules_grammar.lark', rel_to=__file__, parser='lalr', propagate_positions=True), file_name=file_name)
+
+
+if __name__ == '__main__':
+    # This Parser is generally not intended to be run as a stand-alone binary.
+    # The ability to run is for debug and to print out the parsed AST.
+    import click
+    import coloredlogs
+
+    # Supported log levels, mapping string values required for argument
+    # parsing into logging constants
+    __LOG_LEVELS__ = {
+        'debug': logging.DEBUG,
+        'info': logging.INFO,
+        'warn': logging.WARN,
+        'fatal': logging.FATAL,
+    }
+
+    @click.command()
+    @click.option(
+        '--log-level',
+        default='INFO',
+        type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False),
+        help='Determines the verbosity of script output.')
+    @click.argument('filename')
+    def main(log_level, filename=None):
+        coloredlogs.install(level=__LOG_LEVELS__[
+                            log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
+
+        logging.info("Starting to parse ...")
+        data = CreateParser(filename).parse()
+        logging.info("Parse completed")
+
+        logging.info("Data:")
+        logging.info("%r" % data)
+
+    main()