Add C++ API docs to documentation builds (#414)

- Sphinx documentation builds are now a cmake target
- exhale & breathe extensions integrate doxygen generated output into Sphinx docs
- Marking this as breaking since this PR changes the way the documentation is built

New documentation procedure is:

1. Configure cmake with: `MORPHEUS_BUILD_DOCS=ON`
1. `cmake --build build --target morpheus_docs`


Closes #272

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #414

.gitignore

@@ -189,6 +189,7 @@ dask-worker-space
 *.lock
 triton_models/**/*.onnx
 triton_models/**/*.engine
+docs/source/_lib
 docs/source/_modules
 data/*
 mlruns/*

CMakeLists.txt

@@ -30,6 +30,7 @@ option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build p
 option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg.
   Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF)
 option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
+option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
 set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory")
 
 set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
@@ -99,6 +100,10 @@ if(MORPHEUS_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()
 
+if(MORPHEUS_BUILD_DOCS)
+  add_subdirectory(docs)
+endif()
+
 # Get all of the python files to copy to the build directory
 file(GLOB_RECURSE MORPHEUS_PYTHON_FILES
   RELATIVE ${PROJECT_SOURCE_DIR}

ci/scripts/github/docs.sh

@@ -32,16 +32,17 @@ cd ${MORPHEUS_ROOT}
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs
 
-cd ${MORPHEUS_ROOT}/docs
 rapids-logger "Installing Documentation dependencies"
-pip install -r requirement.txt
+mamba env update -f ${MORPHEUS_ROOT}/docs/conda_docs.yml
 
-rapids-logger "Building docs"
+rapids-logger "Configuring for docs"
+cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_BUILD_DOCS=ON .
 
-make -j ${PARALLEL_LEVEL} html
+rapids-logger "Building docs"
+cmake --build build --target morpheus_docs
 
-rapids-logger "Tarring the docs"
-tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/html
+rapids-logger "Archiving the docs"
+tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
 aws s3 cp --no-progress "${WORKSPACE_TMP}/docs.tar.bz" "${ARTIFACT_URL}/docs.tar.bz"

cmake/FindSphinx.cmake

@@ -0,0 +1,29 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+# Based on example in:
+# https://devblogs.microsoft.com/cppblog/clear-functional-c-documentation-with-sphinx-breathe-doxygen-cmake/
+
+#Look for an executable called sphinx-build
+find_program(SPHINX_EXECUTABLE
+             NAMES sphinx-build
+             DOC "Path to sphinx-build executable")
+
+include(FindPackageHandleStandardArgs)
+
+#Handle standard arguments to find_package like REQUIRED and QUIET
+find_package_handle_standard_args(Sphinx
+                                  "Failed to find sphinx-build executable"
+                                  SPHINX_EXECUTABLE)

docker/conda/environments/requirements.txt

@@ -9,6 +9,7 @@ ipywidgets
 jupyter-core>=4.11.2,<5.0
 jupyterlab
 nvidia-pyindex
+pytest-benchmark>=4.0
 torch==1.10.2+cu113
 tritonclient[all]==2.17.*
 websockets

docs/CMakeLists.txt

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(APPEND CMAKE_MESSAGE_CONTEXT "docs")
+
+find_package(Doxygen REQUIRED dot doxygen)
+find_package(Sphinx REQUIRED)
+
+set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR}/html)
+
+add_custom_target(${PROJECT_NAME}_docs ALL
+                  COMMAND
+                  BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}
+                  ${SPHINX_EXECUTABLE} -b html -j auto -T
+                  ${SPHINX_SOURCE} ${SPHINX_BUILD}
+                  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                  COMMENT "Generating documentation with Sphinx")
+
+list(POP_BACK CMAKE_MESSAGE_CONTEXT)

docs/README.md

@@ -1,13 +1,13 @@
 <!--
  SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  SPDX-License-Identifier: Apache-2.0
- 
+
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
- 
+
  http://www.apache.org/licenses/LICENSE-2.0
- 
+
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,19 +17,17 @@
 
 # Building Documentation
 
-A basic python environment with packages listed in `./requirement.txt` is
-enough to build the docs.
+Additional packages required for building the documentation are defined in `./conda_docs.yml`.
 
 ## Get additional dependency
-
+From the root of the morpheus repo:
 ```bash
-pip install -r requirement.txt
+mamba env update -f docs/conda_docs.yml
 ```
 
-## Run makefile:
-
-```bash
-make html
+## To build the documentation run:
 ```
-
-Outputs to `build/html/index.html`
+cmake -B build -DMORPHEUS_BUILD_DOCS=ON .
+cmake --build build --target morpheus_docs
+```
+Outputs to `build/docs/html`

docs/conda_docs.yml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: morpheus
+channels:
+    - conda-forge
+dependencies:
+    - doxygen=1.9.2
+    - pip
+    ####### Morpheus Pip Dependencies (keep sorted!) #######
+    - pip:
+        # Ensure all runtime requirements are installed using the requirements file
+        - --requirement requirements.txt

docs/requirement.txt → docs/requirements.txt

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+breathe==4.34.0
+exhale==0.3.6
 ipython
 myst-parser==0.17.2
 nbsphinx

docs/source/api.rst

@@ -17,11 +17,8 @@
 API Reference
 =============
 
-This page contains the list of project's modules.
+.. toctree::
+   :maxdepth: 3
 
-.. autosummary::
-   :toctree: _modules
-   :template: custom-module-template.rst
-   :recursive:
-
-   morpheus
+   py_api
+   _lib/index

docs/source/conf.py

@@ -30,6 +30,7 @@
 import importlib
 import os
 import sys
+import textwrap
 
 import packaging
 
@@ -71,18 +72,60 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    "IPython.sphinxext.ipython_console_highlighting",
-    "IPython.sphinxext.ipython_directive",
+    'breathe',
+    'exhale',
+    'IPython.sphinxext.ipython_console_highlighting',
+    'IPython.sphinxext.ipython_directive',
     'myst_parser',
-    "nbsphinx",
+    'nbsphinx',
     'numpydoc',
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',
     'sphinx.ext.doctest',
+    'sphinx.ext.graphviz',
     'sphinx.ext.intersphinx',
     'sphinx.ext.linkcode',
 ]
 
+# Breathe Configuration
+breathe_default_project = "morpheus"
+
+# This will be set when invoked by cmake
+build_dir = os.environ.get('BUILD_DIR', './')
+doxygen_tmp_dir = os.path.join(build_dir, "_doxygen/xml")
+breathe_projects = {"morpheus": doxygen_tmp_dir}
+
+exhale_args = {
+    "containmentFolder":
+        "./_lib",
+    "rootFileName":
+        "index.rst",
+    "doxygenStripFromPath":
+        "../../",
+    "rootFileTitle":
+        "C++ API",
+    "createTreeView":
+        True,
+    "exhaleExecutesDoxygen":
+        True,
+    "exhaleDoxygenStdin":
+        textwrap.dedent('''
+        BRIEF_MEMBER_DESC = YES
+        BUILTIN_STL_SUPPORT = YES
+        DOT_IMAGE_FORMAT = svg
+        EXCLUDE_PATTERNS = */tests/* */include/nvtext/* */__pycache__/*
+        EXCLUDE_SYMBOLS = "@85" "cudf*" "py::literals" "RdKafka" "srf*" "std*"
+        EXTENSION_MAPPING = cu=C++ cuh=C++
+        EXTRACT_ALL = YES
+        FILE_PATTERNS = *.c *.cc *.cpp *.h *.hpp *.cu *.cuh *.md
+        HAVE_DOT = YES
+        HIDE_UNDOC_MEMBERS = NO
+        INPUT = ../../morpheus/_lib
+        INTERACTIVE_SVG = YES
+        SOURCE_BROWSER = YES
+    ''')
+}
+
 # Include Python objects as they appear in source files
 # Default: alphabetically ('alphabetical')
 # autodoc_member_order = 'groupwise'