Generate reST documentation for magic commands (#113)

Auto generate docs for any added magics by searching through the source files for lines with register_line_cell_magic, capturing the names for those magics, and calling them inside an ipython kernel with the -h argument, then storing that output into a generated datalab.magics.rst file.

Author: yebrahim

.gitignore

@@ -7,3 +7,4 @@ MANIFEST
 build
 .coverage
 dist
+datalab.magics.rst

docs/Makefile

@@ -55,38 +55,42 @@ help:
 clean:
 	rm -rf $(BUILDDIR)/*
 
-html:
+pre-build:
+	@echo "Generate reST for magic commands:"
+	ipython gen-magic-rst.ipy
+
+html: pre-build
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: pre-build
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: pre-build
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: pre-build
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: pre-build
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: pre-build
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: pre-build
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -95,15 +99,15 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/api.qhc"
 
-applehelp:
+applehelp: pre-build
 	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
 	@echo
 	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
 	@echo "N.B. You won't be able to view it unless you put it in" \
 	      "~/Library/Documentation/Help or install it in your application" \
 	      "bundle."
 
-devhelp:
+devhelp: pre-build
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -112,85 +116,85 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/api"
 	@echo "# devhelp"
 
-epub:
+epub: pre-build
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-latexpdfja:
+latexpdfja: pre-build
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through platex and dvipdfmx..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: pre-build
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: pre-build
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-texinfo:
+texinfo: pre-build
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo
 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."
 
-info:
+info: pre-build
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
 
-gettext:
+gettext: pre-build
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
-changes:
+changes: pre-build
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: pre-build
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-doctest:
+doctest: pre-build
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
-coverage:
+coverage: pre-build
 	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
 	@echo "Testing of coverage in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/coverage/python.txt."
 
-xml:
+xml: pre-build
 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
 	@echo
 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
 
-pseudoxml:
+pseudoxml: pre-build
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
@@ -202,7 +206,7 @@ prepublish:
 	cd ../../datalab-docs && git clone https://github.com/GoogleCloudPlatform/datalab.git html && \
 		git checkout gh-pages
 
-publish:
+publish: pre-build
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	cd ../../datalab-docs/html && git add . && git commit -m "Updated" && git push --force origin gh-pages
 

docs/README

@@ -1,9 +1,8 @@
-To use, install the prerequisites:
+To use, install the prerequisites and the pydatalab module:
 
   pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon
+  pip install .. # from docs directory
 
+then in the docs directory, do 'make html' (or epub, or text, etc).
 
-then in the docs directory, do 'make html' (or epub, or pdf, etc).
-
-Output will be in the docs/_build directory.
-
+Output will be in $BUILDDIR, defaulting to ../../datalab-docs.

docs/conf.py

@@ -145,7 +145,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = []
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/gen-magic-rst.ipy

@@ -0,0 +1,39 @@
+import subprocess, pkgutil, importlib, sys
+from cStringIO import StringIO
+
+# ignore mlalpha and tensorboard for now because of their tensorflow dependency
+# until tensorboard is pip installable and can be listed as a pydatalab dependency
+IGNORED_MAGICS = ['mlalpha', 'tensorboard']
+
+# import submodules
+submodules = [s for _,s,_ in pkgutil.iter_modules(['../datalab'])]
+
+for m in submodules:
+  name = 'datalab.' + m + '.commands'
+  try:
+    importlib.import_module(name)
+  except:
+    sys.stderr.write('WARNING, could not find module ' + name + '. Ignoring..\n')
+
+magic_regex = "find ../datalab -name '*.py' -exec perl -e '$f=join(\"\",<>); print \"$1\n\" if $f=~/register_line_cell_magic\ndef ([^\(]+)/m' {} \;"
+magics = subprocess.check_output(magic_regex, shell=True)
+
+reSTfile = open('datalab.magics.rst', 'w')
+indent = '\n  '
+
+reSTfile.write('datalab.magics\n')
+reSTfile.write('=================\n\n')
+
+for m in magics.split():
+  if m in IGNORED_MAGICS:
+    sys.stderr.write('Ignoring magic ' + m + '\n')
+  else:
+    reSTfile.write('.. attribute:: ' + m + '\n')
+    reSTfile.write('.. parsed-literal::\n')
+    # hijack stdout since the ipython kernel call writes to stdout/err directly
+    # and does not return its output
+    tmpStdout, sys.stdout = sys.stdout, StringIO()
+    get_ipython().magic(m + ' -h')
+    resultout = sys.stdout.getvalue().splitlines()
+    sys.stdout = tmpStdout
+    reSTfile.writelines(indent + indent.join(resultout) + '\n\n')

docs/index.rst

@@ -16,6 +16,7 @@ Contents:
    datalab.data
    datalab.stackdriver.monitoring
    datalab.storage
+   datalab.magics
 
 
 Indices and tables

setup.py

@@ -79,8 +79,8 @@
     'pytz>=2015.4',
     'pyyaml==3.11',
     'requests==2.9.1',
-    'scikit-learn==0.17.1',
     'scipy==0.18.0',
+    'scikit-learn==0.17.1',
     'ipykernel==4.4.1',
   ],
   package_data={