Merge pull request #1988 from catmaid/sphinx-qol

Sphinx quality of life improvements

.travis.yml

@@ -29,7 +29,7 @@ before_install:
 install:
   # Install requirements for running CATMAID and its unit tests
   - ./scripts/travis/install_requirements.sh
-  - pip install coveralls flake8
+  - pip install coveralls
   - npm install
 before_script:
   # Set up and start postgres, create database, write config files

Vagrantfile

@@ -14,11 +14,15 @@ Vagrant.configure("2") do |config|
   #config.vm.box = "ubuntu/bionic64"
   config.vm.box = "bento/ubuntu-18.04"
   config.vm.synced_folder "./", "/CATMAID"
-  
+
   config.vm.hostname = "catmaid-vm"
   config.vm.define "catmaid-vm"
 
+  # django dev server
   config.vm.network "forwarded_port", guest: 8888, host: 8888, host_ip: "127.0.0.1"
+  # HTML sphinx-docs with `make serve`
+  config.vm.network "forwarded_port", guest: 8889, host: 8889, host_ip: "127.0.0.1"
+  # postgreSQL
   config.vm.network "forwarded_port", guest: 5432, host: 5555, host_ip: "127.0.0.1"
   config.vm.network "private_network", type: "dhcp"
 

django/requirements-doc.txt

@@ -5,3 +5,4 @@ Markdown==2.6.9
 Pygments==2.2.0
 Sphinx==2.4.2
 sphinxcontrib-images==0.9.1
+watchdog[watchmedo]==0.10.2

django/requirements-test.txt

@@ -3,3 +3,4 @@ django-migration-testcase==0.0.13
 selenium==3.141.0
 sauceclient==1.0.0
 mypy==0.761; platform_python_implementation != 'PyPy'
+flake8==3.7.9

sphinx-doc/Makefile

@@ -12,7 +12,9 @@ PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
+.PHONY: default help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest watch
+
+default: html
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -36,50 +38,63 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
+clean-%:
+	rm -rf $(BUILDDIR)/$*/*
+
 apidoc:
 	@echo
 	@echo "Requires local CATMAID server on port 8000"
-	`coreapi dump --format openapi > source/_static/api/openapi.json`
+	`coreapi dump --format openapi > source/_static/api/openapi.json` && \
 	`npm bin`/redoc-cli bundle source/_static/api/openapi.json -o source/_static/api/index.html
 
 widgetdoc:
 	@echo
 	@echo "Collecting HTML widget documentation."
-	`rm -f -r source/_static/widgets`
+	`rm -f -r source/_static/widgets` && \
 	`cp -r ../django/applications/catmaid/static/html/doc/widgets source/_static/widgets`
 
-html:
+serve: clean-html
+	mkdir -p $(BUILDDIR)/html && cd $(BUILDDIR)/html && python -m http.server 8889 &
+	$(MAKE) watch-html
+
+watch: watch-html
+
+watch-%:
+	$(MAKE) $*
+	watchmedo shell-command --debug-force-polling --recursive --wait --pattern "*.rst;*.png;*.svg;*.jpg" --command '$(MAKE) $*' source/
+
+html: clean-html
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: clean-dirhtml
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: clean-singlehtml
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: clean-pickle
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: clean-json
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: clean-htmlhelp
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: clean-qthelp
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -88,7 +103,7 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CATMAID.qhc"
 
-devhelp:
+devhelp: clean-devhelp
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -97,46 +112,46 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CATMAID"
 	@echo "# devhelp"
 
-epub:
+epub: clean-epub
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: clean-latex
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: clean-latex
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	make -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: clean-text
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: clean-man
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-changes:
+changes: clean-changes
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: clean-linkcheck
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-doctest:
+doctest: clean-doctest
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."

sphinx-doc/source/contributing.rst

@@ -343,6 +343,8 @@ ci.org/catmaid/CATMAID/branches>`_, several automated processes are performed
 to help verify the correctness and quality of CATMAID:
 
 * :doc:`Unit and integration tests for Django backend <djangounittest>`
+* Linting (static analysis) of the python code with flake8
+* Type checking of the python code with mypy
 * Linting (static analysis) of the javascript code with JSHint
 * Linting of CSS with csslint
 * Unit tests of javascript code with QUnit
@@ -358,6 +360,12 @@ Django tests are run through Django's admin commands::
         cd /<path_to_catmaid_install>/django/projects
         ./manage.py test catmaid.tests
 
+flake8 and mypy are installed along with other python development dependencies.
+Run them with::
+
+    flake8 django
+    mypy django
+
 JSHint can be `installed from NPM or your platform's package manager
 <http://jshint.com/install/>`_ and should use CATMAID's config settings::
 
@@ -410,12 +418,13 @@ ReStructured Text. Documentation from commits pushed to the official CATMAID
 repository are built by `Read the Docs <https://readthedocs.org/>`_ and hosted
 at `catmaid.org <http://catmaid.org>`_.
 
-To build the general documentation from within your pip virtualenv, run::
-
-    cd sphinx-doc
-    make html
+CATMAID's documentation can be built in various formats
+by navigating to the ``sphinx-doc`` directory and using `make <https://www.gnu.org/software/make/>`_.
+The default (i.e. ``make`` with no arguments) is HTML, which builds the documentation at sphinx-doc/build/html/index.html.
 
-The built documentation is now in ``sphinx-doc/build/html/index.html``.
+Every build target can automatically be built when the source files change,
+by using ``make watch-<target>``.
+Build the HTML docs, watch for changes, and serve the documentation at ``http://localhost:8889`` using ``make serve``.
 
 In-Client Documentation
 #######################

sphinx-doc/source/vagrant.rst

@@ -53,7 +53,7 @@ Once installed, you need only ``vagrant up`` inside the repository to create a f
 Tools like VScode need to be able to SSH into the container themselves.
 They get their configuration from your user's SSH configuration (``~/.ssh/config`` on Linux).
 Copy and paste the output of ``vagrant ssh-config`` (ignoring any warnings from ruby gems) into this file.
-The hostname of the container will be ``catmaid-vm``. 
+The hostname of the container will be ``catmaid-vm``.
 
 .. warning::
 
@@ -71,8 +71,17 @@ In Linux, ``/`` is the root directory, and ``~`` is the home directory of the us
 * R packages are in ``~/R``
 * Data is written to ``~/data``
 
-The virtual machine also forwards the ports that PostgreSQL and the Django development server listen on, so that you can access them as if they were local to the host.
-To avoid clashing with the host, these are forwarded to 5555 and 8888 respectively.
+Some guest ports are forwarded to the host machine so that you can access the database, test with the dev server, and look at the generated sphinx docs.
+
++-------------------+------------+-----------+---------------------------------------------------+
+| Service           | Guest port | Host port | Notes                                             |
++===================+============+===========+===================================================+
+| PostgreSQL        | 5432       | 5555      |                                                   |
++-------------------+------------+-----------+---------------------------------------------------+
+| Django dev server | 8888       | 8888      | ``django/projects/manage.py runserver [::]:8888`` |
++-------------------+------------+-----------+---------------------------------------------------+
+| Docs server       | 8889       | 8889      | ``cd sphinx-doc && make serve``                   |
++-------------------+------------+-----------+---------------------------------------------------+
 
 Setup
 -----