codegen.py: allow generators to take input arguments (#26147)

* Pass in custom options to code generators (e.g. package names, generator filters)

* Fix typo and add impl example for idl_plugin example

* Add documentation for option passing

* Restyle

* Fix unit tests

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>

scripts/codegen.py

@@ -68,12 +68,16 @@ def write_new_data(self, relative_path: str, content: str):
     type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False),
     help='Determines the verbosity of script output')
 @click.option(
-    '--generator',
+    '--generator', '-g',
     default='java-jni',
     help='What code generator to run.  The choices are: '+'|'.join(GENERATORS.keys())+'. ' +
          'When using custom, provide the plugin path using `--generator custom:<path_to_plugin>:<plugin_module_name>` syntax. ' +
-    'For example, `--generator custom:./my_plugin:my_plugin_module` will load `./my_plugin/my_plugin_module/__init.py__` ' +
+         'For example, `--generator custom:./my_plugin:my_plugin_module` will load `./my_plugin/my_plugin_module/__init.py__` ' +
          'that defines a subclass of CodeGenerator named CustomGenerator.')
+@click.option(
+    '--option',
+    multiple=True,
+    help="Extra generator options, of the form: --option key:value")
 @click.option(
     '--output-dir',
     type=click.Path(exists=False),
@@ -97,7 +101,7 @@ def write_new_data(self, relative_path: str, content: str):
 @click.argument(
     'idl_path',
     type=click.Path(exists=True))
-def main(log_level, generator, output_dir, dry_run, name_only, expected_outputs, idl_path):
+def main(log_level, generator, option, output_dir, dry_run, name_only, expected_outputs, idl_path):
     """
     Parses MATTER IDL files (.matter) and performs SDK code generation
     as set up by the program arguments.
@@ -121,19 +125,28 @@ def main(log_level, generator, output_dir, dry_run, name_only, expected_outputs,
     idl_tree = CreateParser().parse(open(idl_path, "rt").read())
 
     plugin_module = None
-    if generator.startswith('custom'):
+    if generator.startswith('custom:'):
         # check that the plugin path is provided
-        if ':' not in generator:
-            logging.fatal("Custom generator plugin path not provided. Use --generator custom:<path_to_plugin>")
-            sys.exit(1)
         custom_params = generator.split(':')
+        if len(custom_params) != 3:
+            logging.fatal("Custom generator format not valid. Please use --generator custom:<path>:<module>")
+            sys.exit(1)
         (generator, plugin_path, plugin_module) = custom_params
+
         logging.info("Using CustomGenerator at plugin path %s.%s" % (plugin_path, plugin_module))
         sys.path.append(plugin_path)
         generator = 'CUSTOM'
 
+    extra_args = {}
+    for o in option:
+        if ':' not in o:
+            logging.fatal("Please specify options as '<key>:<value>'. %r is not valid. " % o)
+            sys.exit(1)
+        key, value = o.split(':')
+        extra_args[key] = value
+
     logging.info("Running code generator %s" % generator)
-    generator = CodeGenerator.FromString(generator).Create(storage, idl=idl_tree, plugin_module=plugin_module)
+    generator = CodeGenerator.FromString(generator).Create(storage, idl=idl_tree, plugin_module=plugin_module, **extra_args)
     generator.render(dry_run)
 
     if expected_outputs:

scripts/py_matter_idl/examples/README.md

@@ -13,9 +13,13 @@ scripts/py_matter_idl/examples/matter_idl_plugin:
    "matter_idl_plugin" subdirectory.
 4. Execute the `codegen.py` script passing the path to the parent directory of
    "matter_idl_plugin" via
-   `--generator custom:<plugin_path>:<plugin_module_name>` argument.
+   `--generator custom:<plugin_path>:<plugin_module_name>` argument and package
+   name like `--option package:com.example.matter.proto`
 
 ```
 # From top-of-tree in this example
-./scripts/codegen.py --generator custom:./scripts/py_matter_idl/examples:matter_idl_plugin ./src/controller/data_model/controller-clusters.matter
+./scripts/codegen.py \
+  --generator custom:./scripts/py_matter_idl/examples:matter_idl_plugin \
+  --option package:com.example.matter.proto \
+  ./src/controller/data_model/controller-clusters.matter
 ```

scripts/py_matter_idl/examples/matter_idl_plugin/__init__.py

@@ -200,6 +200,10 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs):
         """
         super().__init__(storage, idl, fs_loader_searchpath=os.path.dirname(__file__))
 
+        if 'package' not in kargs:
+            raise Exception('Please provide a "--option package:<name>" argument')
+        self.package = kargs['package']
+
         # String helpers
         self.jinja_env.filters['toLowerSnakeCase'] = toLowerSnakeCase
         self.jinja_env.filters['toUpperSnakeCase'] = toUpperSnakeCase
@@ -238,5 +242,6 @@ def internal_render_all(self):
                 output_file_name=filename,
                 vars={
                     'cluster': cluster,
+                    'package': self.package,
                 }
             )

scripts/py_matter_idl/examples/matter_idl_plugin/matter_cluster_proto.jinja

@@ -3,7 +3,7 @@
 
 syntax = "proto3";
 
-package com.matter.example.proto;
+package {{package}};
 
 option java_multiple_files = true;
 

scripts/py_matter_idl/matter_idl/test_generators.py

@@ -128,7 +128,7 @@ def _create_generator(self, storage: GeneratorStorage, idl: Idl):
             sys.path.append(os.path.abspath(
                 os.path.join(os.path.dirname(__file__), '../examples')))
             from matter_idl_plugin import CustomGenerator
-            return CustomGenerator(storage, idl)
+            return CustomGenerator(storage, idl, package='com.matter.example.proto')
         else:
             raise Exception("Unknown generator for testing: %s",
                             self.generator_name.lower())