Skip to content

Commit

Permalink
wrapper updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@varunagrawal
varunagrawal committed Jul 11, 2021
1 parent e8e3094 commit fe95b8b
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 94 deletions.
157 changes: 82 additions & 75 deletions wrap/cmake/PybindWrap.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,14 @@ gtwrap_get_python_version(${WRAP_PYTHON_VERSION})
message(STATUS "Setting Python version for wrapper")
set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})

# User-friendly Pybind11 wrapping and installing function.
# Builds a Pybind11 module from the provided interface_header.
# For example, for the interface header gtsam.h, this will
# build the wrap module 'gtsam_py.cc'.
# User-friendly Pybind11 wrapping and installing function. Builds a Pybind11
# module from the provided interface_headers. For example, for the interface
# header gtsam.h, this will build the wrap module 'gtsam_py.cc'.
#
# Arguments:
# ~~~
# target: The Make target
# interface_header: The relative path to the wrapper interface definition file.
# interface_headers: List of paths to the wrapper interface definition files. The top level interface file should be first.
# generated_cpp: The name of the cpp file which is generated from the tpl file.
# module_name: The name of the Python module to use.
# top_namespace: The C++ namespace under which the code to be wrapped exists.
Expand All @@ -31,16 +30,17 @@ set(PYBIND11_PYTHON_VERSION ${WRAP_PYTHON_VERSION})
# libs: Libraries to link with.
# dependencies: Dependencies which need to be built before the wrapper.
# use_boost (optional): Flag indicating whether to include Boost.
function(pybind_wrap
target
interface_header
generated_cpp
module_name
top_namespace
ignore_classes
module_template
libs
dependencies)
function(
pybind_wrap
target
interface_headers
generated_cpp
module_name
top_namespace
ignore_classes
module_template
libs
dependencies)
set(ExtraMacroArgs ${ARGN})
list(GET ExtraMacroArgs 0 USE_BOOST)
if(USE_BOOST)
Expand All @@ -49,57 +49,62 @@ function(pybind_wrap
set(_WRAP_BOOST_ARG "")
endif(USE_BOOST)

if (UNIX)
if(UNIX)
set(GTWRAP_PATH_SEPARATOR ":")
else()
set(GTWRAP_PATH_SEPARATOR ";")
endif()

add_custom_command(OUTPUT ${generated_cpp}
COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
${PYTHON_EXECUTABLE}
${PYBIND_WRAP_SCRIPT}
--src
${interface_header}
--out
${generated_cpp}
--module_name
${module_name}
--top_module_namespaces
"${top_namespace}"
--ignore
${ignore_classes}
--template
${module_template}
${_WRAP_BOOST_ARG}
DEPENDS ${interface_header} ${module_template}
VERBATIM)
add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${generated_cpp})
# Convert .i file names to .cpp file names.
foreach(filepath ${interface_headers})
get_filename_component(interface ${filepath} NAME)
string(REPLACE ".i" ".cpp" cpp_file ${interface})
list(APPEND cpp_files ${cpp_file})
endforeach()

add_custom_command(
OUTPUT ${cpp_files}
COMMAND
${CMAKE_COMMAND} -E env
"PYTHONPATH=${GTWRAP_PACKAGE_DIR}${GTWRAP_PATH_SEPARATOR}$ENV{PYTHONPATH}"
${PYTHON_EXECUTABLE} ${PYBIND_WRAP_SCRIPT} --src "${interface_headers}"
--out "${generated_cpp}" --module_name ${module_name}
--top_module_namespaces "${top_namespace}" --ignore ${ignore_classes}
--template ${module_template} ${_WRAP_BOOST_ARG}
DEPENDS "${interface_headers}" ${module_template}
VERBATIM)

add_custom_target(pybind_wrap_${module_name} ALL DEPENDS ${cpp_files})

# Late dependency injection, to make sure this gets called whenever the
# interface header or the wrap library are updated.
# ~~~
# See: https://stackoverflow.com/questions/40032593/cmake-does-not-rebuild-dependent-after-prerequisite-changes
# ~~~
add_custom_command(OUTPUT ${generated_cpp}
DEPENDS ${interface_header}
# @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
# @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
# @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
APPEND)
add_custom_command(
OUTPUT ${cpp_files}
DEPENDS ${interface_headers}
# @GTWRAP_SOURCE_DIR@/gtwrap/interface_parser.py
# @GTWRAP_SOURCE_DIR@/gtwrap/pybind_wrapper.py
# @GTWRAP_SOURCE_DIR@/gtwrap/template_instantiator.py
APPEND)

pybind11_add_module(${target} ${generated_cpp})
pybind11_add_module(${target} "${cpp_files}")

if(APPLE)
# `type_info` objects will become "weak private external" if the templated class is initialized implicitly even if we explicitly
# export them with `WRAP_EXPORT`. If that happens, the `type_info` for the same templated class will diverge between shared
# libraries, causing `dynamic_cast` to fail. This is mitigated by telling Clang to mimic the MSVC behavior.
# See https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
# `type_info` objects will become "weak private external" if the templated
# class is initialized implicitly even if we explicitly export them with
# `WRAP_EXPORT`. If that happens, the `type_info` for the same templated
# class will diverge between shared libraries, causing `dynamic_cast` to
# fail. This is mitigated by telling Clang to mimic the MSVC behavior. See
# https://developer.apple.com/library/archive/technotes/tn2185/_index.html#//apple_ref/doc/uid/DTS10004200-CH1-SUBSECTION2
# https://github.com/CppMicroServices/CppMicroServices/pull/82/files
# https://www.russellmcc.com/posts/2013-08-03-rtti.html
target_compile_options(${target} PRIVATE "-fvisibility-ms-compat")
endif()

add_dependencies(${target} pybind_wrap_${module_name})

if(NOT "${libs}" STREQUAL "")
target_link_libraries(${target} PRIVATE "${libs}")
endif()
Expand All @@ -121,10 +126,7 @@ endfunction()
# dest_directory: The destination directory to install to.
# patterns: list of file patterns to install
# ~~~
function(install_python_scripts
source_directory
dest_directory
patterns)
function(install_python_scripts source_directory dest_directory patterns)
set(patterns_args "")
set(exclude_patterns "")

Expand All @@ -144,17 +146,19 @@ function(install_python_scripts
# there is one
get_filename_component(location "${dest_directory}" PATH)
get_filename_component(name "${dest_directory}" NAME)
install(DIRECTORY "${source_directory}"
DESTINATION "${location}/${name}${build_type_tag}"
CONFIGURATIONS "${build_type}"
FILES_MATCHING ${patterns_args}
PATTERN "${exclude_patterns}" EXCLUDE)
install(
DIRECTORY "${source_directory}"
DESTINATION "${location}/${name}${build_type_tag}"
CONFIGURATIONS "${build_type}"
FILES_MATCHING ${patterns_args}
PATTERN "${exclude_patterns}" EXCLUDE)
endforeach()
else()
install(DIRECTORY "${source_directory}"
DESTINATION "${dest_directory}"
FILES_MATCHING ${patterns_args}
PATTERN "${exclude_patterns}" EXCLUDE)
install(
DIRECTORY "${source_directory}"
DESTINATION "${dest_directory}"
FILES_MATCHING ${patterns_args}
PATTERN "${exclude_patterns}" EXCLUDE)
endif()

endfunction()
Expand All @@ -172,13 +176,14 @@ function(install_python_files source_files dest_directory)
foreach(build_type ${CMAKE_CONFIGURATION_TYPES})
string(TOUPPER "${build_type}" build_type_upper)
set(build_type_tag "")
# Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if
# there is one
# Split up filename to strip trailing '/' in WRAP_PY_INSTALL_PATH if there
# is one
get_filename_component(location "${dest_directory}" PATH)
get_filename_component(name "${dest_directory}" NAME)
install(FILES "${source_files}"
DESTINATION "${location}/${name}${build_type_tag}"
CONFIGURATIONS "${build_type}")
install(
FILES "${source_files}"
DESTINATION "${location}/${name}${build_type_tag}"
CONFIGURATIONS "${build_type}")
endforeach()
else()
install(FILES "${source_files}" DESTINATION "${dest_directory}")
Expand All @@ -194,18 +199,19 @@ function(create_symlinks source_folder dest_folder)
return()
endif()

file(GLOB files
LIST_DIRECTORIES true
RELATIVE "${source_folder}"
"${source_folder}/*")
file(
GLOB files
LIST_DIRECTORIES true
RELATIVE "${source_folder}"
"${source_folder}/*")
foreach(path_file ${files})
get_filename_component(folder ${path_file} PATH)
get_filename_component(ext ${path_file} EXT)
set(ignored_ext ".tpl" ".h")
list (FIND ignored_ext "${ext}" _index)
if (${_index} GREATER -1)
list(FIND ignored_ext "${ext}" _index)
if(${_index} GREATER -1)
continue()
endif ()
endif()
# Create REAL folder
file(MAKE_DIRECTORY "${dest_folder}")

Expand All @@ -224,9 +230,10 @@ function(create_symlinks source_folder dest_folder)
endif()
# cmake-format: on

execute_process(COMMAND ${command}
RESULT_VARIABLE result
ERROR_VARIABLE output)
execute_process(
COMMAND ${command}
RESULT_VARIABLE result
ERROR_VARIABLE output)

if(NOT ${result} EQUAL 0)
message(
Expand Down
75 changes: 67 additions & 8 deletions wrap/gtwrap/pybind_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# pylint: disable=too-many-arguments, too-many-instance-attributes, no-self-use, no-else-return, too-many-arguments, unused-format-string-argument, line-too-long

import re
from pathlib import Path

import gtwrap.interface_parser as parser
import gtwrap.template_instantiator as instantiator
Expand All @@ -32,7 +33,7 @@ def __init__(self,
self.top_module_namespaces = top_module_namespaces
self.use_boost = use_boost
self.ignore_classes = ignore_classes
self._serializing_classes = list()
self._serializing_classes = []
self.module_template = module_template
self.python_keywords = [
'lambda', 'False', 'def', 'if', 'raise', 'None', 'del', 'import',
Expand Down Expand Up @@ -160,7 +161,7 @@ def _wrap_method(self,
'self->print',
'py::scoped_ostream_redirect output; self->print')

# Make __repr__() call print() internally
# Make __repr__() call .print() internally
ret += '''{prefix}.def("__repr__",
[](const {cpp_class}& self{opt_comma}{args_signature_with_names}){{
gtsam::RedirectCout redirect;
Expand Down Expand Up @@ -557,8 +558,15 @@ def wrap_namespace(self, namespace):
)
return wrapped, includes

def wrap(self, content):
"""Wrap the code in the interface file."""
def wrap_file(self, content, module_name=None, submodules=None):
"""
Wrap the code in the interface file.
Args:
content: The contents of the interface file.
module_name: The name of the module.
submodules: List of other interface file names that should be linked to.
"""
# Parse the contents of the interface file
module = parser.Module.parseString(content)
# Instantiate all templates
Expand All @@ -574,23 +582,74 @@ def wrap(self, content):
if ',' in cpp_class:
new_name = re.sub("[,:<> ]", "", cpp_class)
boost_class_export += "typedef {cpp_class} {new_name};\n".format( # noqa
cpp_class=cpp_class,
new_name=new_name,
)
cpp_class=cpp_class, new_name=new_name)

boost_class_export += "BOOST_CLASS_EXPORT({new_name})\n".format(
new_name=new_name, )

# Reset the serializing classes list
self._serializing_classes = []

holder_type = "PYBIND11_DECLARE_HOLDER_TYPE(TYPE_PLACEHOLDER_DONOTUSE, " \
"{shared_ptr_type}::shared_ptr<TYPE_PLACEHOLDER_DONOTUSE>);"
include_boost = "#include <boost/shared_ptr.hpp>" if self.use_boost else ""

submodules_init = []

if submodules is not None:
module_def = "PYBIND11_MODULE({0}, m_)".format(module_name)

for idx, submodule in enumerate(submodules):
submodules[idx] = "void {0}(py::module_ &);".format(submodule)
submodules_init.append("{0}(m_);".format(submodule))

else:
module_def = "void {0}(py::module_ &m_)".format(module_name)
submodules = []

return self.module_template.format(
include_boost=include_boost,
module_name=self.module_name,
module_def=module_def,
module_name=module_name,
includes=includes,
holder_type=holder_type.format(
shared_ptr_type=('boost' if self.use_boost else 'std'))
if self.use_boost else "",
wrapped_namespace=wrapped_namespace,
boost_class_export=boost_class_export,
submodules="\n".join(submodules),
submodules_init="\n".join(submodules_init),
)

def wrap(self, sources, main_output):
"""
Wrap all the source interface files.
Args:
sources: List of all interface files.
main_output: The name for the main module.
"""
main_module = sources[0]
submodules = []
for source in sources[1:]:
filename = Path(source).name
module_name = Path(source).stem
# Read in the complete interface (.i) file
with open(source, "r") as f:
content = f.read()
submodules.append(module_name)
cc_content = self.wrap_file(content, module_name=module_name)

# Generate the C++ code which Pybind11 will use.
with open(filename.replace(".i", ".cpp"), "w") as f:
f.write(cc_content)

with open(main_module, "r") as f:
content = f.read()
cc_content = self.wrap_file(content,
module_name=self.module_name,
submodules=submodules)

# Generate the C++ code which Pybind11 will use.
with open(main_output, "w") as f:
f.write(cc_content)
11 changes: 2 additions & 9 deletions wrap/scripts/pybind_wrap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,6 @@ def main():
if top_module_namespaces[0]:
top_module_namespaces = [''] + top_module_namespaces

# Read in the complete interface (.i) file
with open(args.src, "r") as f:
content = f.read()

with open(args.template, "r") as f:
template_content = f.read()

Expand All @@ -83,11 +79,8 @@ def main():
)

# Wrap the code and get back the cpp/cc code.
cc_content = wrapper.wrap(content)

# Generate the C++ code which Pybind11 will use.
with open(args.out, "w") as f:
f.write(cc_content)
sources = args.src.split(';')
wrapper.wrap(sources, args.out)


if __name__ == "__main__":
Expand Down
Loading

0 comments on commit fe95b8b

Please sign in to comment.