Polish CPP docs, Minor Python Docs Fixes (pytorch#11722)

Differential Revision: D9919120

Pulled By: goldsborough

fbshipit-source-id: bf14cbe4ab79524495957cb749828046af864aab

.gitignore

@@ -25,9 +25,8 @@ aten/src/ATen/cuda/CUDAConfig.h
 build/
 dist/
 docs/src/**/*
-docs/cpp/xml/
-docs/cpp/html/
-docs/cpp/api/
+docs/cpp/build
+docs/cpp/source/api
 test/.coverage
 test/cpp/api/mnist
 test/custom_operator/model.pt

.travis.yml

@@ -28,4 +28,4 @@ matrix:
         script: mypy @mypy-files.txt
       - env: CPP_DOC_CHECK
         install: sudo apt-get install -y doxygen
-        script: cd docs/cpp && ./check-doxygen.sh
+        script: cd docs/cpp/source && ./check-doxygen.sh

docs/Makefile

@@ -39,4 +39,4 @@ html-stable:
 
 clean:
 	@echo "Removing everything under 'build'.."
-	@rm -r $(BUILDDIR)/html/ $(BUILDDIR)/doctrees
+	@rm -rf $(BUILDDIR)/html/ $(BUILDDIR)/doctrees

docs/cpp/Makefile

@@ -4,21 +4,25 @@
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = PyTorch
-SOURCEDIR     = ./
+SOURCEDIR     = source
 BUILDDIR      = build
 PYCMD         = python
 
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help Makefile
+.PHONY: help Makefile clean
+
+clean:
+	@# Clean up sphinx and doxygen build artifacts.
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@# Clean up exhale generated api.
+	@echo "Removing everything under generated 'source/api'..."
+	@rm -rf $(SOURCEDIR)/api
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-clean:
-	@echo "Removing everything under 'build'..."
-	@rm -r $(BUILDDIR)/html/ $(BUILDDIR)/doctrees $(BUILDDIR)/xml

docs/cpp/source/Doxyfile

@@ -0,0 +1,132 @@
+################################################################################
+# Primary project setup.                                                       #
+################################################################################
+PROJECT_NAME           = "PyTorch"
+# Parent directory of where Doxygen should output files.  Cannot be for example
+# `../build/doxygen` because if `../build` does not exist Doxygen will error.
+OUTPUT_DIRECTORY       = ../build
+# XML output stored in ${OUTPUT_DIRECTORY}/${XML_OUTPUT}.  Must agree
+# with what is specified to `breathe_projects` in `conf.py`.
+XML_OUTPUT             = xml
+# HTML output stored in ${OUTPUT_DIRECTORY}/${HTML_OUTPUT}.
+# NOTE: it can be useful to have Doxygen generate HTML to verify what sphinx is
+#       creating, or to generate HTML documentation to check a docstring you are
+#       writing without needing to wait for Sphinx.
+#
+#       # Must run from this directory (see relative paths below...)
+#       $ cd /path/to/pytorch/docs/cpp/source
+#
+#       # Send configs in this file plus `GENERATE_HTML=YES` to doxygen on stdin
+#       $ (cat Doxyfile; echo 'GENERATE_HTML = YES') | doxygen -
+#
+#       # View the doxygen results.
+#       $ open ../build/doxygen_html/index.html
+#
+#       This sends everything in this file plus `GENERATE_HTML = YES` to doxygen
+#       on stdin.  Without needing to edit this `Doxyfile` directly.
+HTML_OUTPUT            = doxygen_html
+# Strip the path prefix.  *MUST* agree with `exhale_args` in conf.py.
+# {repo_root}/docs/cpp/source/../../.. -> {repo_root}
+STRIP_FROM_PATH        = ../../..
+# What folders / files Doxygen should process.
+INPUT                  = ../../../torch/csrc/api/include \
+                         ../../../torch/csrc/api/src \
+                         ../../../torch/csrc/jit/custom_operator.h \
+                         ../../../torch/csrc/jit/import.h \
+                         ../../../torch/csrc/jit/ivalue.h \
+                         ../../../torch/csrc/jit/script/module.h \
+                         ../../../aten/src/ATen/ATen.h \
+                         ../../../aten/src/ATen/Backend.h \
+                         ../../../aten/src/ATen/Device.h \
+                         ../../../aten/src/ATen/DeviceGuard.h \
+                         ../../../aten/src/ATen/Layout.h \
+                         ../../../aten/src/ATen/OptionsGuard.h \
+                         ../../../aten/src/ATen/Scalar.h \
+                         ../../../aten/src/ATen/TensorOptions.h \
+                         ../../../aten/src/ATen/core/ArrayRef.h \
+                         ../../../aten/src/ATen/core/DeviceType.h \
+                         ../../../aten/src/ATen/core/Error.h \
+                         ../../../aten/src/ATen/core/Half.h \
+                         ../../../aten/src/ATen/core/ScalarType.h \
+                         ../../../aten/src/ATen/cuda/CUDAGuard.h \
+                         ../../../aten/src/ATen/cuda/CUDAStream.h \
+                         ../../../aten/src/ATen/cuda/CUDAContext.h \
+                         ../../../aten/src/ATen/cudnn/Descriptors.h \
+                         ../../../aten/src/ATen/cudnn/Handles.h \
+                         ../../../aten/src/ATen/cudnn/Types.h \
+                         ../../../aten/src/ATen/cudnn/Utils.h \
+                         ../../../aten/src/ATen/mkl/Descriptors.h \
+                         ../../../build/aten/src/ATen/Tensor.h \
+                         ../../../build/aten/src/ATen/Functions.h
+# Don't include .cpp files!
+FILE_PATTERNS          = *.h
+# If you need this to be YES, exhale will probably break.
+CREATE_SUBDIRS         = NO
+# So that Doxygen does not trim paths, which affects the file hierarchy
+FULL_PATH_NAMES        = YES
+# Nested folders will be ignored without this.
+RECURSIVE              = YES
+################################################################################
+# Output formats for Doxygen to create.                                        #
+################################################################################
+# Set to YES if you are debugging or want to compare.
+GENERATE_HTML          = NO
+# Unless you want it...
+GENERATE_LATEX         = NO
+# Both breathe and exhale need the xml.
+GENERATE_XML           = YES
+# Set to NO if you do not want the Doxygen program listing included.
+# NOTE: setting to NO may result in unrecovered file relationships
+#       (which file defined which compound)
+XML_PROGRAMLISTING     = YES
+################################################################################
+# Doxygen preprocessor / parser control.                                       #
+################################################################################
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = NO
+SKIP_FUNCTION_MACROS   = NO
+# Extra defs for to help with building the _right_ version of the docs
+PREDEFINED             = DOXYGEN_DOCUMENTATION_BUILD
+PREDEFINED            += DOXYGEN_SHOULD_SKIP_THIS
+# Symbol expansion to follow for #include statements (not input)
+SEARCH_INCLUDES        = YES
+INCLUDE_PATH           = ../../.. \
+                         ../../../torch/csrc/api/include \
+                         ../../../aten/src \
+                         ../../../build/aten/src
+################################################################################
+# Compound extraction control.                                                 #
+################################################################################
+EXTRACT_ALL            = YES
+EXTRACT_PACKAGE        = YES
+EXTRACT_STATIC         = YES
+CASE_SENSE_NAMES       = NO
+EXCLUDE_SYMBOLS        = c10::* caffe2::* cereal* DL* TH* cudnn*
+################################################################################
+# Docstring control / customization.                                           #
+################################################################################
+# First line of /** comment */ treated as \brief
+JAVADOC_AUTOBRIEF      = YES
+# Allow for rst directives and advanced functions e.g. grid tables
+# Example:
+#     /**
+#      * \rst
+#      * .. code-block:: cpp
+#      *
+#      *     int main() {
+#      *         return 0;
+#      *     }
+#      *
+#      * \endrst
+#      */
+# NOTE:
+#     1. \rst and \endrst must be on their own line.
+#     2. leading-asterisk required.
+ALIASES                = "rst=\verbatim embed:rst:leading-asterisk"
+ALIASES               += "endrst=\endverbatim"
+################################################################################
+# Warning suppression.                                                         #
+################################################################################
+QUIET                  = YES
+WARN_IF_UNDOCUMENTED   = NO

docs/cpp/check-doxygen.sh → docs/cpp/source/check-doxygen.sh

@@ -4,7 +4,9 @@ set -ex
 
 ignore_warning() {
   # Invert match to filter out $1.
+  set +e
   grep -v "$1" doxygen-log.txt > temp.txt
+  set -e
   mv temp.txt doxygen-log.txt
 }
 
@@ -17,11 +19,7 @@ cat original-doxygen-log.txt
 
 # Filter out some warnings.
 ignore_warning "warning: no uniquely matching class member found for"
-ignore_warning "warning: source ../../build/aten/src/ is not a readable file"
-ignore_warning "warning: source ../../build/aten/src/ATen/Tensor.h is not a readable file"
-ignore_warning "warning: source ../../build/aten/src/ATen/Functions.h is not a readable file"
-ignore_warning "warning: documented symbol \`torch::nn::FunctionalImpl::FunctionalImpl' was not declared or defined"
-ignore_warning "functional.h:81: warning: Found ';' while parsing initializer list!"
+ignore_warning "warning:.*\.\./\.\./\.\./build/aten.*"
 
 # Count the number of remaining warnings.
 warnings="$(grep 'warning:' doxygen-log.txt | wc -l)"

docs/cpp/conf.py → docs/cpp/source/conf.py

@@ -17,7 +17,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
+import os
 # sys.path.insert(0, os.path.abspath('.'))
 
 import sys
@@ -29,7 +29,7 @@
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #
-# needs_sphinx = '1.0'
+needs_sphinx = '1.6'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -39,7 +39,24 @@
     'exhale'
 ]
 
-breathe_projects = {"PyTorch": "build/xml"}
+# Setup absolute paths for communicating with breathe / exhale where
+# items are expected / should be trimmed by.
+# This file is {repo_root}/docs/cpp/source/conf.py
+this_file_dir = os.path.abspath(os.path.dirname(__file__))
+doxygen_xml_dir = os.path.join(
+    os.path.dirname(this_file_dir),  # {repo_root}/docs/cpp
+    'build',                         # {repo_root}/docs/cpp/build
+    'xml'                            # {repo_root}/docs/cpp/build/xml
+)
+repo_root = os.path.dirname(  # {repo_root}
+    os.path.dirname(          # {repo_root}/docs
+        os.path.dirname(      # {repo_root}/docs/cpp
+            this_file_dir     # {repo_root}/docs/cpp/source
+        )
+    )
+)
+
+breathe_projects = {"PyTorch": doxygen_xml_dir}
 breathe_default_project = "PyTorch"
 
 # Setup the exhale extension
@@ -50,7 +67,7 @@
     "containmentFolder": "./api",
     "rootFileName": "library_root.rst",
     "rootFileTitle": "Library API",
-    "doxygenStripFromPath": "../",
+    "doxygenStripFromPath": repo_root,
     ############################################################################
     # Suggested optional arguments.                                            #
     ############################################################################
@@ -137,7 +154,6 @@
 # a list of builtin themes.
 #
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -150,21 +166,33 @@
     'logo_only': True,
 }
 
-html_logo = '../source/_static/img/pytorch-logo-dark-unstable.png'
+# NOTE: sharing python docs resources
+html_logo = os.path.join(
+    repo_root, 'docs', 'source', '_static', 'img', 'pytorch-logo-dark-unstable.png'
+)
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['../source/_static']
+# NOTE: sharing python docs resources
+html_static_path = [os.path.join(repo_root, 'docs', 'source', '_static')]
 
-# html_style_path = 'css/pytorch_theme.css'
-html_context = {
-    'css_files': [
-        'https://fonts.googleapis.com/css?family=Lato',
-        '_static/css/pytorch_theme.css'
-    ],
-}
 
+# Called automatically by Sphinx, making this `conf.py` an "extension".
+def setup(app):
+    # NOTE: in Sphinx 1.8+ `html_css_files` is an official configuration value
+    # and can be moved outside of this function (and the setup(app) function
+    # can be deleted).
+    html_css_files = [
+        'https://fonts.googleapis.com/css?family=Lato',
+        'css/pytorch_theme.css'  # relative to paths in `html_static_path`
+    ]
+
+    # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is
+    # `add_stylesheet` (deprecated in 1.8).
+    add_css = getattr(app, 'add_css_file', getattr(app, 'add_stylesheet'))
+    for css_file in html_css_files:
+        add_css(css_file)
 
 # -- Options for HTMLHelp output ------------------------------------------
 

docs/source/conf.py

@@ -38,7 +38,7 @@
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #
-# needs_sphinx = '1.0'
+needs_sphinx = '1.6'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -135,7 +135,6 @@
 #
 #
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -159,14 +158,23 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static', '_images']
 
-html_style_path = 'css/pytorch_theme.css'
-html_context = {
-    'css_files': [
+
+# Called automatically by Sphinx, making this `conf.py` an "extension".
+def setup(app):
+    # NOTE: in Sphinx 1.8+ `html_css_files` is an official configuration value
+    # and can be moved outside of this function (and the setup(app) function
+    # can be deleted).
+    html_css_files = [
         'https://fonts.googleapis.com/css?family=Lato',
-        '_static/css/pytorch_theme.css',
-        'https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css',
-    ],
-}
+        'css/pytorch_theme.css',  # relative to paths in `html_static_path`
+        'https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css'
+    ]
+
+    # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is
+    # `add_stylesheet` (deprecated in 1.8).
+    add_css = getattr(app, 'add_css_file', getattr(app, 'add_stylesheet'))
+    for css_file in html_css_files:
+        add_css(css_file)
 
 
 # -- Options for HTMLHelp output ------------------------------------------

torch/csrc/api/include/torch/nn/module.h

@@ -14,12 +14,15 @@
 #include <type_traits>
 #include <unordered_map>
 
+// forward declarations confuse doxygen
+#ifndef DOXYGEN_SHOULD_SKIP_THIS
 namespace torch {
 namespace detail {
 template <typename T>
 class CursorBase;
 } // namespace detail
 } // namespace torch
+#endif // DOXYGEN_SHOULD_SKIP_THIS
 
 namespace torch {
 namespace nn {