Add Python codegen (#306)

This changeset also drops support for Python 3.6.

Fixes OpenCyphal/pycyphal#110
Required for OpenCyphal/pycyphal#277
Supersedes #234

.github/workflows/test.yml

@@ -219,3 +219,16 @@ jobs:
                           --toolchain-family ${{ matrix.compiler }} \
                           --endianness ${{ matrix.endianness }} \
                           ${{ matrix.flag }}
+
+  language-verification-python:
+    runs-on: ubuntu-latest
+    needs: test
+    container: ghcr.io/opencyphal/toxic:tx22.4.1
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: verify
+      run: |
+        cd verification/python
+        nox

.readthedocs.yaml

@@ -0,0 +1,8 @@
+version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+python:
+   install:
+   - requirements: requirements.txt

.vscode/settings.json

@@ -39,6 +39,7 @@
     "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools",
     "files.associations": {
         "*.py.template": "python",
+        "nunavut_support.j2": "python",
         "*.cc": "cpp",
         "*.hpp": "cpp",
         "__bit_reference": "cpp",
@@ -211,4 +212,55 @@
         "/dsdl/i",
         "/bitspan/"
     ],
+    "cSpell.words": [
+        "allclose",
+        "astype",
+        "autouse",
+        "bitorder",
+        "bools",
+        "builtins",
+        "Bxxx",
+        "caplog",
+        "CDEF",
+        "codegen",
+        "Cyphal",
+        "doctests",
+        "DSDL",
+        "dtype",
+        "EDCB",
+        "elementwise",
+        "emptylines",
+        "endianness",
+        "errstate",
+        "fillvalue",
+        "fpid",
+        "frombuffer",
+        "htmlcov",
+        "itemsize",
+        "Kirienko",
+        "maxsplit",
+        "nbytes",
+        "ndarray",
+        "ndim",
+        "nnvg",
+        "noxfile",
+        "opencyphal",
+        "outdir",
+        "packbits",
+        "Pavel",
+        "postprocessor",
+        "postprocessors",
+        "pycyphal",
+        "pydsdl",
+        "roadmap",
+        "Sriram",
+        "tobytes",
+        "transcompilation",
+        "typecheck",
+        "uavcan",
+        "unpackbits",
+        "unseparate",
+        "unstropped",
+        "WKCV"
+    ],
 }

CONTRIBUTING.rst

@@ -188,7 +188,7 @@ We rely on `read the docs`_ to build our documentation from github but we also v
 as part of our tox build. This means you can view a local copy after completing a full, successful
 test run (See `Running The Tests`_) or do
 :code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.1 /bin/sh -c "tox -e docs"` to build
-the docs target. You can open the index.html under .tox/docs/tmp/index.html or run a local
+the docs target. You can open the index.html under ``.tox/docs/tmp/index.html`` or run a local
 web-server::
 
     python3 -m http.server --directory .tox/docs/tmp &

README.rst

@@ -36,13 +36,9 @@ and it can be used to generate code for other languages if custom templates (and
 Currently, the following languages are supported out of the box:
 
 - **C11** (generates header-only libraries)
-- **HTML** (generates documentation pages) (experimental support)
-
-The following languages are currently on the roadmap:
-
-- **Python** (already supported in `Pycyphal`_, pending
-  `transplantation into Nunavut <https://github.com/OpenCyphal/pycyphal/issues/110>`_)
-- **C++ 14 and newer** (generates header-only libraries; `work-in-progress <https://github.com/OpenCyphal/nunavut/issues/91>`_)
+- **C++** (generates header-only libraries; `work-in-progress <https://github.com/OpenCyphal/nunavut/issues/91>`_)
+- **Python** (generates Python packages)
+- **HTML** (generates documentation pages)
 
 Nunavut is named after the `Canadian territory`_. We chose the name because it
 is a beautiful word to say and read.
@@ -88,6 +84,17 @@ documentation sections.
     nnvg --experimental-languages --target-language html public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan
     nnvg --experimental-languages --target-language html public_regulated_data_types/uavcan
 
+Generate Python packages using the command-line tool
+----------------------------------------------------
+
+This example assumes that the public regulated namespace directories ``reg`` and ``uavcan`` reside under
+``public_regulated_data_types/``.
+Nunavut is invoked to generate code for the former.
+
+.. code-block:: shell
+
+    nnvg --target-language py public_regulated_data_types/reg --lookup-dir public_regulated_data_types/uavcan
+
 
 Use custom templates
 --------------------

conf.py

@@ -28,7 +28,7 @@
 # The full version, including alpha/beta/rc tags
 release = nunavut_version
 
-exclude_patterns = ["**/test"]
+exclude_patterns = ["**/test", "**/.nox"]
 
 with open(".gitignore", "r") as gif:
     for line in gif:
@@ -67,7 +67,7 @@
 source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = "docs/index"
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

conftest.py

@@ -397,6 +397,7 @@ def mock_environment(request):  # type: ignore
         "**/CONTRIBUTING.rst",
         "**/verification/*",
         "**/prof/*",
+        "*.j2",
         "*.png",
     ],
     fixtures=[

docs/languages.rst

@@ -39,3 +39,37 @@ nnvg command
         --experimental-languages \
         -I path/to/public_regulated_data_types/uavcan \
         /path/to/my_types
+
+*************************
+Python
+*************************
+
+The Python language support generates Python packages that depend on the following packages:
+
+* **PyDSDL** --- maintained by the OpenCyphal team at https://github.com/OpenCyphal/pydsdl.
+* **NumPy** --- a third-party dependency needed for fast serialization of arrays, esp. bit arrays.
+* :code:`nunavut_support.py` --- produced by Nunavut itself and stored next to the other generated packages.
+  When redistributing generated code, this package should be included as well.
+
+These are the only dependencies of the generated code. Nunavut itself is notably excluded from this list.
+The generated code should be compatible with all current versions of Python.
+To see the specific versions of Python and dependencies that generated code is tested against,
+please refer to ``verification/python`` in the source tree.
+
+At the moment there are no code generation options for Python;
+that is, the generated code is always the same irrespective of the options given.
+
+The ``nunavut_support.py`` module includes several members that are useful for working with generated code.
+The documentation for each member is provided in the docstrings of the module itself;
+please be sure to read it.
+The most important members are:
+
+* :code:`serialize`, :code:`deserialize` --- (de)serialize a DSDL object.
+* :code:`get_model`, :code:`get_class` --- map a Python class to a PyDSDL AST model and vice versa.
+* :code:`get_extent_bytes`, :code:`get_fixed_port_id`, etc. --- get information about a DSDL object.
+* :code:`to_builtin`, :code:`update_from_builtin` --- convert a DSDL object to/from a Python dictionary.
+  This is useful for conversion between DSDL and JSON et al.
+* :code:`get_attribute`, :code:`set_attribute` --- get/set object fields.
+  DSDL fields that are named like Python builtins or keywords are modified with a trailing underscore;
+  .e.g., ``if`` becomes ``if_``.
+  These helpers allow one to access fields by their DSDL name without having to worry about this.

index.rst

@@ -0,0 +1,13 @@
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+
+   docs/api/library
+   docs/languages
+   docs/templates
+   CLI (nnvg) <docs/cli>
+   docs/dev
+   Appendix <docs/appendix>
+
+.. include:: README.rst

requirements.txt

@@ -1,4 +1,5 @@
 # This file provided for readthedocs.io only. Use tox.ini for all dependencies.
 
+.
 sphinx-argparse
 sphinxemoji

setup.cfg

@@ -16,7 +16,6 @@ classifiers =
     License :: OSI Approved :: MIT License
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
@@ -44,7 +43,7 @@ install_requires=
 
 zip_safe = False
 
-python_requires = >=3.6
+python_requires = >=3.7
 
 [options.entry_points]
 console_scripts =

src/nunavut/_version.py

@@ -8,7 +8,7 @@
 .. autodata:: __version__
 """
 
-__version__ = "2.1.1"
+__version__ = "2.2.0"
 __license__ = "MIT"
 __author__ = "OpenCyphal"
 __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright (c) 2023 OpenCyphal."

src/nunavut/jinja/loaders.py

@@ -199,7 +199,7 @@ def type_to_template(self, value_type: typing.Type) -> typing.Optional[pathlib.P
             template_name = l.type_to_template(pydsdl.StructureType)
 
             assert template_name is not None
-            assert template_name.name == 'Any.j2'
+            assert template_name.name == 'StructureType.j2'
 
         """
         template_path = None

src/nunavut/lang/properties.yaml

@@ -332,6 +332,8 @@ nunavut.lang.py:
     enable_stropping: true
     encoding_prefix: zX
     stropping_suffix: _
+    limit_empty_lines: 1
+    trim_trailing_whitespace: true
 
 nunavut.lang.js:
     extension: .js

src/nunavut/lang/py/__init__.py

@@ -7,10 +7,15 @@
     Filters for generating python. All filters in this
     module will be available in the template's global namespace as ``py``.
 """
+from __future__ import annotations
 import builtins
 import functools
 import keyword
-import typing
+import base64
+import gzip
+import pickle
+import itertools
+from typing import Any, Iterable
 
 import pydsdl
 
@@ -31,7 +36,11 @@ class Language(BaseLanguage):
     Concrete, Python-specific :class:`nunavut.lang.Language` object.
     """
 
-    PYTHON_RESERVED_IDENTIFIERS = sorted(list(map(str, list(keyword.kwlist) + dir(builtins))))  # type: typing.List[str]
+    PYTHON_RESERVED_IDENTIFIERS: list[str] = sorted(list(map(str, list(keyword.kwlist) + dir(builtins))))
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._language_options["enable_serialization_asserts"] = True
 
     @functools.lru_cache(maxsize=None)
     def _get_token_encoder(self) -> TokenEncoder:
@@ -40,11 +49,11 @@ def _get_token_encoder(self) -> TokenEncoder:
         """
         return TokenEncoder(self, additional_reserved_identifiers=self.PYTHON_RESERVED_IDENTIFIERS)
 
-    def get_includes(self, dep_types: Dependencies) -> typing.List[str]:
+    def get_includes(self, dep_types: Dependencies) -> list[str]:
         # imports aren't includes
         return []
 
-    def filter_id(self, instance: typing.Any, id_type: str = "any") -> str:
+    def filter_id(self, instance: Any, id_type: str = "any") -> str:
         raw_name = self.default_filter_id_for_target(instance)
 
         return self._get_token_encoder().strop(raw_name, id_type)
@@ -108,7 +117,7 @@ def filter_to_template_unique_name(context: SupportsTemplateContext, base_token:
 
 
 @template_language_filter(__name__)
-def filter_id(language: Language, instance: typing.Any, id_type: str = "any") -> str:
+def filter_id(language: Language, instance: Any, id_type: str = "any") -> str:
     """
     Filter that produces a valid Python identifier for a given object. The encoding may not
     be reversible.
@@ -264,7 +273,7 @@ def filter_short_reference_name(language: Language, t: pydsdl.CompositeType) ->
 
 
 @template_language_list_filter(__name__)
-def filter_imports(language: Language, t: pydsdl.CompositeType, sort: bool = True) -> typing.List[str]:
+def filter_imports(language: Language, t: pydsdl.CompositeType, sort: bool = True) -> list[str]:
     """
     Returns a list of all modules that must be imported to use a given type.
 
@@ -302,7 +311,7 @@ def array_w_composite_type(data_type: pydsdl.Any) -> bool:
 
 
 @template_language_int_filter(__name__)
-def filter_longest_id_length(language: Language, attributes: typing.List[pydsdl.Attribute]) -> int:
+def filter_longest_id_length(language: Language, attributes: list[pydsdl.Attribute]) -> int:
     """
     Return the length of the longest identifier in a list of :class:`pydsdl.Attribute` objects.
 
@@ -332,3 +341,52 @@ def filter_longest_id_length(language: Language, attributes: typing.List[pydsdl.
         return max(map(len, map(functools.partial(filter_id, language), attributes)))
     else:
         return max(map(len, attributes))
+
+
+def filter_pickle(x: Any) -> str:
+    """
+    Serializes the given object using pickle and then compresses it using gzip and then encodes it using base85.
+    """
+    pck = base64.b85encode(gzip.compress(pickle.dumps(x, protocol=4))).decode().strip()  # type: str
+    segment_gen = map("".join, itertools.zip_longest(*([iter(pck)] * 100), fillvalue=""))
+    return "\n".join(repr(x) for x in segment_gen)
+
+
+def filter_numpy_scalar_type(t: pydsdl.Any) -> str:
+    """
+    Returns the numpy scalar type that is the closest match to the given DSDL type.
+    """
+
+    def pick_width(w: int) -> int:
+        for o in [8, 16, 32, 64]:
+            if w <= o:
+                return o
+        raise ValueError(f"Invalid bit width: {w}")  # pragma: no cover
+
+    if isinstance(t, pydsdl.BooleanType):
+        return "_np_.bool_"
+    if isinstance(t, pydsdl.SignedIntegerType):
+        return f"_np_.int{pick_width(t.bit_length)}"
+    if isinstance(t, pydsdl.UnsignedIntegerType):
+        return f"_np_.uint{pick_width(t.bit_length)}"
+    if isinstance(t, pydsdl.FloatType):
+        return f"_np_.float{pick_width(t.bit_length)}"
+    assert not isinstance(t, pydsdl.PrimitiveType), "Forgot to handle some primitive types"
+    return "_np_.object_"
+
+
+def filter_newest_minor_version_aliases(tys: Iterable[pydsdl.CompositeType]) -> list[tuple[str, pydsdl.CompositeType]]:
+    """
+    Implementation of https://github.com/OpenCyphal/nunavut/issues/193
+    """
+    tys = list(tys)
+    return [
+        (
+            f"{name}_{major}",
+            max(
+                (t for t in tys if t.short_name == name and t.version.major == major),
+                key=lambda x: int(x.version.minor),
+            ),
+        )
+        for name, major in sorted({(x.short_name, x.version.major) for x in tys})
+    ]