Squashed 'wrap/' changes from 07330d100..d9ae5ce03

d9ae5ce03 Merge pull request borglab#118 from borglab/feature/matlab-multi-files 9adddf7dd update the main script for matlab wrapping 0b0398d46 remove debug statements since they aren't needed for now df064a364 support for parsing mutiple interface files for Matlab wrapping 1929e197c add test for parsing multiple interface files bac442056 Merge pull request borglab#117 from borglab/fix/matlab-refactor 331f4a8ce update tests to remove redundant code 5426e3af4 generate all content from within the wrap function f78612bf9 make directory check common b7acd7a1f fixed import and test setup 88007b153 Merge pull request borglab#116 from borglab/feature/matlab-refactor a074896e6 utils -> mixins 414557e00 structure 187100439 update gitignore adbc55aea don't use class attributes in matlab wrapper f45ba5b2d broke down some large functions into smaller ones 7756f0548 add mixin for checks and call method to wrap global functions a318e2a67 Merge pull request borglab#115 from borglab/feature/multiple-modules b02b74c3d convert matlab_wrapper to a submodule be8641e83 improved function naming in tests 02ddbfbb0 update tests and docs dfbded2c7 small fixes e9ec5af07 update docs d124e2cfb wrap multiple files 7c7342f86 update cmake to take in new changes for multiple modules 54850f724 Merge pull request borglab#114 from borglab/fix/remove-py35 71ee98321 add mypy annotations ccaea6294 remove support for python 3.5 git-subtree-dir: wrap git-subtree-split: d9ae5ce036c4315db3c28b12db9c73eae246f314
varunagrawal · Jul 11, 2021 · 4c410fc · 4c410fc
1 parent 56bede0
commit 4c410fc
Show file tree

Hide file tree

Showing 41 changed files with 1,424 additions and 1,633 deletions.
diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
       - name: Checkout

diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,4 @@ __pycache__/
 # Files related to code coverage stats
 **/.coverage
 
-gtwrap/matlab_wrapper.tpl
+gtwrap/matlab_wrapper/matlab_wrapper.tpl
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -58,7 +58,7 @@ if(NOT DEFINED GTWRAP_INCLUDE_NAME)
 endif()
 
 configure_file(${PROJECT_SOURCE_DIR}/templates/matlab_wrapper.tpl.in
-               ${PROJECT_SOURCE_DIR}/gtwrap/matlab_wrapper.tpl)
+               ${PROJECT_SOURCE_DIR}/gtwrap/matlab_wrapper/matlab_wrapper.tpl)
 
 # Install the gtwrap python package as a directory so it can be found  by CMake
 # for wrapping.

diff --git a/DOCS.md b/DOCS.md
@@ -192,12 +192,14 @@ The python wrapper supports keyword arguments for functions/methods. Hence, the
 
    - **DO NOT** re-define an overriden function already declared in the external (forward-declared) base class. This will cause an ambiguity problem in the Pybind header file.
 
+- Splitting wrapper over multiple files
+    - The Pybind11 wrapper supports splitting the wrapping code over multiple files.
+    - To be able to use classes from another module, simply import the C++ header file in that wrapper file.
+    - Unfortunately, this means that aliases can no longer be used.
+    - Similarly, there can be multiple `preamble.h` and `specializations.h` files. Each of these should match the module file name.
 
 ### TODO
-- Default values for arguments.
-    - WORKAROUND: make multiple versions of the same function for different configurations of default arguments.
 - Handle `gtsam::Rot3M` conversions to quaternions.
 - Parse return of const ref arguments.
 - Parse `std::string` variants and convert directly to special string.
-- Add enum support.
 - Add generalized serialization support via `boost.serialization` with hooks to MATLAB save/load.
diff --git a/README.md b/README.md
@@ -29,8 +29,10 @@ Using `wrap` in your project is straightforward from here. In your `CMakeLists.t
 ```cmake
 find_package(gtwrap)
 
+set(interface_files ${PROJECT_SOURCE_DIR}/cpp/${PROJECT_NAME}.h)
+
 pybind_wrap(${PROJECT_NAME}_py # target
-            ${PROJECT_SOURCE_DIR}/cpp/${PROJECT_NAME}.h # interface header file
+            "${interface_files}" # list of interface header files
             "${PROJECT_NAME}.cpp" # the generated cpp
             "${PROJECT_NAME}" # module_name
             "${PROJECT_MODULE_NAME}" # top namespace in the cpp file e.g. gtsam

diff --git a/cmake/PybindWrap.cmake b/cmake/PybindWrap.cmake
@@ -13,15 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION})
 message(STATUS "Setting Python version for wrapper")
 set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
 
-# User-friendly Pybind11 wrapping and installing function.
-# Builds a Pybind11 module from the provided interface_header.
-# For example, for the interface header gtsam.h, this will
-# build the wrap module 'gtsam_py.cc'.
+# User-friendly Pybind11 wrapping and installing function. Builds a Pybind11
+# module from the provided interface_headers. For example, for the interface
+# header gtsam.h, this will build the wrap module 'gtsam_py.cc'.
 #
 # Arguments:
 # ~~~
 # target: The Make target
-# interface_header:  The relative path to the wrapper interface definition file.
+# interface_headers:  List of paths to the wrapper interface definition files. The top level interface file should be first.
 # generated_cpp: The name of the cpp file which is generated from the tpl file.
 # module_name: The name of the Python module to use.
 # top_namespace: The C++ namespace under which the code to be wrapped exists.
@@ -31,16 +30,17 @@ set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
 # libs: Libraries to link with.
 # dependencies: Dependencies which need to be built before the wrapper.
 # use_boost (optional): Flag indicating whether to include Boost.
-function(pybind_wrap
-         target
-         interface_header
-         generated_cpp
-         module_name
-         top_namespace
-         ignore_classes
-         module_template
-         libs
-         dependencies)
+function(
+  pybind_wrap
+  target
+  interface_headers
+  generated_cpp
+  module_name
+  top_namespace
+  ignore_classes
+  module_template
+  libs
+  dependencies)
   set(ExtraMacroArgs ${ARGN})
   list(GET ExtraMacroArgs 0 USE_BOOST)
   if(USE_BOOST)
@@ -49,57 +49,62 @@ function(pybind_wrap
     set(_WRAP_BOOST_ARG "")
   endif(USE_BOOST)
 
-  if (UNIX)
+  if(UNIX)
     set(GTWRAP_PATH_SEPARATOR ":")
   else()
     set(GTWRAP_PATH_SEPARATOR ";")
   endif()
 
-  add_custom_command(OUTPUT ${generated_cpp}
-          COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
-          ${PYTHON_EXECUTABLE}
-                             ${PYBIND_WRAP_SCRIPT}
-                             --src
-                             ${interface_header}
-                             --out
-                             ${generated_cpp}
-                             --module_name
-                             ${module_name}
-                             --top_module_namespaces
-                             "${top_namespace}"
-                             --ignore
-                             ${ignore_classes}
-                             --template
-                             ${module_template}
-                             ${_WRAP_BOOST_ARG}
-                     DEPENDS ${interface_header} ${module_template}
-                     VERBATIM)
-  add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${generated_cpp})
+  # Convert .i file names to .cpp file names.
+  foreach(filepath ${interface_headers})
+    get_filename_component(interface ${filepath} NAME)
+    string(REPLACE ".i" ".cpp" cpp_file ${interface})
+    list(APPEND cpp_files ${cpp_file})
+  endforeach()
+
+  add_custom_command(
+    OUTPUT ${cpp_files}
+    COMMAND
+      ${CMAKE_COMMAND} -E env
+      "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
+      ${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_headers}"
+      --out "${generated_cpp}" --module_name ${module_name}
+      --top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
+      --template ${module_template} ${_WRAP_BOOST_ARG}
+    DEPENDS "${interface_headers}" ${module_template}
+    VERBATIM)
+
+  add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files})
 
   # Late dependency injection, to make sure this gets called whenever the
   # interface header or the wrap library are updated.
   # ~~~
   # See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes
   # ~~~
-  add_custom_command(OUTPUT ${generated_cpp}
-                     DEPENDS ${interface_header}
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
-                            #  @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
-                     APPEND)
+  add_custom_command(
+    OUTPUT ${cpp_files}
+    DEPENDS ${interface_headers}
+    # @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
+    # @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
+    # @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
+    APPEND)
 
-  pybind11_add_module(${target} ${generated_cpp})
+  pybind11_add_module(${target} "${cpp_files}")
 
   if(APPLE)
-    # `type_info` objects will become "weak private external" if the templated class is initialized implicitly even if we explicitly
-    # export them with `WRAP_EXPORT`. If that happens, the `type_info` for the same templated class will diverge between shared
-    # libraries, causing `dynamic_cast` to fail. This is mitigated by telling Clang to mimic the MSVC behavior.
-    # See https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
+    # `type_info` objects will become "weak private external" if the templated
+    # class is initialized implicitly even if we explicitly export them with
+    # `WRAP_EXPORT`. If that happens, the `type_info` for the same templated
+    # class will diverge between shared libraries, causing `dynamic_cast` to
+    # fail. This is mitigated by telling Clang to mimic the MSVC behavior. See
+    # https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
     # https://github.com/CppMicroServices/CppMicroServices/pull/82/files
     # https://www.russellmcc.com/posts/2013-08-03-rtti.html
     target_compile_options(${target} PRIVATE "-fvisibility-ms-compat")
   endif()
+
   add_dependencies(${target} pybind_wrap_${module_name})
+
   if(NOT "${libs}" STREQUAL "")
     target_link_libraries(${target} PRIVATE "${libs}")
   endif()
@@ -121,10 +126,7 @@ endfunction()
 # dest_directory: The destination directory to install to.
 # patterns: list of file patterns to install
 # ~~~
-function(install_python_scripts
-         source_directory
-         dest_directory
-         patterns)
+function(install_python_scripts source_directory dest_directory patterns)
   set(patterns_args "")
   set(exclude_patterns "")
 
@@ -144,17 +146,19 @@ function(install_python_scripts
       # there is one
       get_filename_component(location "${dest_directory}" PATH)
       get_filename_component(name "${dest_directory}" NAME)
-      install(DIRECTORY "${source_directory}"
-              DESTINATION "${location}/${name}${build_type_tag}"
-              CONFIGURATIONS "${build_type}"
-              FILES_MATCHING ${patterns_args}
-              PATTERN "${exclude_patterns}" EXCLUDE)
+      install(
+        DIRECTORY "${source_directory}"
+        DESTINATION "${location}/${name}${build_type_tag}"
+        CONFIGURATIONS "${build_type}"
+        FILES_MATCHING ${patterns_args}
+        PATTERN "${exclude_patterns}" EXCLUDE)
     endforeach()
   else()
-    install(DIRECTORY "${source_directory}"
-            DESTINATION "${dest_directory}"
-            FILES_MATCHING ${patterns_args}
-            PATTERN "${exclude_patterns}" EXCLUDE)
+    install(
+      DIRECTORY "${source_directory}"
+      DESTINATION "${dest_directory}"
+      FILES_MATCHING ${patterns_args}
+      PATTERN "${exclude_patterns}" EXCLUDE)
   endif()
 
 endfunction()
@@ -172,13 +176,14 @@ function(install_python_files source_files dest_directory)
     foreach(build_type ${CMAKE_CONFIGURATION_TYPES})
       string(TOUPPER "${build_type}" build_type_upper)
       set(build_type_tag "")
-      # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if
-      # there is one
+      # Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if there
+      # is one
       get_filename_component(location "${dest_directory}" PATH)
       get_filename_component(name "${dest_directory}" NAME)
-      install(FILES "${source_files}"
-              DESTINATION "${location}/${name}${build_type_tag}"
-              CONFIGURATIONS "${build_type}")
+      install(
+        FILES "${source_files}"
+        DESTINATION "${location}/${name}${build_type_tag}"
+        CONFIGURATIONS "${build_type}")
     endforeach()
   else()
     install(FILES "${source_files}" DESTINATION "${dest_directory}")
@@ -194,18 +199,19 @@ function(create_symlinks source_folder dest_folder)
     return()
   endif()
 
-  file(GLOB files
-       LIST_DIRECTORIES true
-       RELATIVE "${source_folder}"
-       "${source_folder}/*")
+  file(
+    GLOB files
+    LIST_DIRECTORIES true
+    RELATIVE "${source_folder}"
+    "${source_folder}/*")
   foreach(path_file ${files})
     get_filename_component(folder ${path_file} PATH)
     get_filename_component(ext ${path_file} EXT)
     set(ignored_ext ".tpl" ".h")
-    list (FIND ignored_ext "${ext}" _index)
-    if (${_index} GREATER -1)
+    list(FIND ignored_ext "${ext}" _index)
+    if(${_index} GREATER -1)
       continue()
-    endif ()
+    endif()
     # Create REAL folder
     file(MAKE_DIRECTORY "${dest_folder}")
 
@@ -224,9 +230,10 @@ function(create_symlinks source_folder dest_folder)
     endif()
     # cmake-format: on
 
-    execute_process(COMMAND ${command}
-                    RESULT_VARIABLE result
-                    ERROR_VARIABLE output)
+    execute_process(
+      COMMAND ${command}
+      RESULT_VARIABLE result
+      ERROR_VARIABLE output)
 
     if(NOT ${result} EQUAL 0)
       message(

diff --git a/gtwrap/interface_parser/__init__.py b/gtwrap/interface_parser/__init__.py
@@ -12,7 +12,7 @@
 
 import sys
 
-import pyparsing
+import pyparsing  # type: ignore
 
 from .classes import *
 from .declaration import *

diff --git a/gtwrap/interface_parser/classes.py b/gtwrap/interface_parser/classes.py
@@ -12,7 +12,7 @@
 
 from typing import Iterable, List, Union
 
-from pyparsing import Literal, Optional, ZeroOrMore
+from pyparsing import Literal, Optional, ZeroOrMore  # type: ignore
 
 from .enum import Enum
 from .function import ArgumentList, ReturnType
@@ -233,7 +233,7 @@ def __init__(self,
             self.static_methods = []
             self.properties = []
             self.operators = []
-            self.enums = []
+            self.enums: List[Enum] = []
             for m in members:
                 if isinstance(m, Constructor):
                     self.ctors.append(m)
@@ -274,7 +274,7 @@ def __init__(self,
 
     def __init__(
         self,
-        template: Template,
+        template: Union[Template, None],
         is_virtual: str,
         name: str,
         parent_class: list,
@@ -292,16 +292,16 @@ def __init__(
         if parent_class:
             # If it is in an iterable, extract the parent class.
             if isinstance(parent_class, Iterable):
-                parent_class = parent_class[0]
+                parent_class = parent_class[0]  # type: ignore
 
             # If the base class is a TemplatedType,
             # we want the instantiated Typename
             if isinstance(parent_class, TemplatedType):
-                parent_class = parent_class.typename
+                parent_class = parent_class.typename  # type: ignore
 
             self.parent_class = parent_class
         else:
-            self.parent_class = ''
+            self.parent_class = ''  # type: ignore
 
         self.ctors = ctors
         self.methods = methods

diff --git a/gtwrap/interface_parser/declaration.py b/gtwrap/interface_parser/declaration.py
@@ -10,7 +10,7 @@
 Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert
 """
 
-from pyparsing import CharsNotIn, Optional
+from pyparsing import CharsNotIn, Optional  # type: ignore
 
 from .tokens import (CLASS, COLON, INCLUDE, LOPBRACK, ROPBRACK, SEMI_COLON,
                      VIRTUAL)

diff --git a/gtwrap/interface_parser/enum.py b/gtwrap/interface_parser/enum.py
@@ -10,7 +10,7 @@
 Author: Varun Agrawal
 """
 
-from pyparsing import delimitedList
+from pyparsing import delimitedList  # type: ignore
 
 from .tokens import ENUM, IDENT, LBRACE, RBRACE, SEMI_COLON
 from .type import Typename