From 19e5d79dac6f4c702378d224791d2f8afae886c7 Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 18:59:16 +0000 Subject: [PATCH 1/9] Vagrant: forward 8889 for serving sphinx docs --- Vagrantfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 95e6861ed4..464548f1ee 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -14,11 +14,15 @@ Vagrant.configure("2") do |config| #config.vm.box = "ubuntu/bionic64" config.vm.box = "bento/ubuntu-18.04" config.vm.synced_folder "./", "/CATMAID" - + config.vm.hostname = "catmaid-vm" config.vm.define "catmaid-vm" + # django dev server config.vm.network "forwarded_port", guest: 8888, host: 8888, host_ip: "127.0.0.1" + # HTML sphinx-docs with `make serve` + config.vm.network "forwarded_port", guest: 8889, host: 8889, host_ip: "127.0.0.1" + # postgreSQL config.vm.network "forwarded_port", guest: 5432, host: 5555, host_ip: "127.0.0.1" config.vm.network "private_network", type: "dhcp" From 5549e18a549d10b87fe1388cd2424f58cfa3b1ba Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:00:03 +0000 Subject: [PATCH 2/9] Sphinx: Various quality of life improvements Makefile: - Each build target has a "clean-" variant, to clear away old files - Each build target has a "watch-" variant, to rebuild on changes - "serve" recipe builds and watches for HTML, then serves them on localhost:8889 --- sphinx-doc/Makefile | 53 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/sphinx-doc/Makefile b/sphinx-doc/Makefile index d434fd62e2..5449ace105 100644 --- a/sphinx-doc/Makefile +++ b/sphinx-doc/Makefile @@ -12,7 +12,9 @@ PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest +.PHONY: default help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest watch + +default: html help: @echo "Please use \`make ' where is one of" @@ -36,50 +38,63 @@ help: clean: -rm -rf $(BUILDDIR)/* +clean-%: + rm -rf $(BUILDDIR)/$* + apidoc: @echo @echo "Requires local CATMAID server on port 8000" - `coreapi dump --format openapi > source/_static/api/openapi.json` + `coreapi dump --format openapi > source/_static/api/openapi.json` && \ `npm bin`/redoc-cli bundle source/_static/api/openapi.json -o source/_static/api/index.html widgetdoc: @echo @echo "Collecting HTML widget documentation." - `rm -f -r source/_static/widgets` + `rm -f -r source/_static/widgets` && \ `cp -r ../django/applications/catmaid/static/html/doc/widgets source/_static/widgets` -html: +serve: clean-html + mkdir -p $(BUILDDIR)/html && cd $(BUILDDIR)/html && python -m http.server 8889 & + $(MAKE) watch-html + +watch: watch-html + +watch-%: + $(MAKE) $* + watchmedo shell-command --recursive --wait --pattern "*.rst;*.png;*.svg;*.jpg" --command '$(MAKE) $*' source/ + +html: clean-html $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -dirhtml: +dirhtml: clean-dirhtml $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." -singlehtml: +singlehtml: clean-singlehtml $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." -pickle: +pickle: clean-pickle $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." -json: +json: clean-json $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: +htmlhelp: clean-htmlhelp $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: +qthelp: clean-qthelp $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -88,7 +103,7 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CATMAID.qhc" -devhelp: +devhelp: clean-devhelp $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @@ -97,46 +112,46 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CATMAID" @echo "# devhelp" -epub: +epub: clean-epub $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." -latex: +latex: clean-latex $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." -latexpdf: +latexpdf: clean-latex $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." make -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -text: +text: clean-text $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." -man: +man: clean-man $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." -changes: +changes: clean-changes $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." -linkcheck: +linkcheck: clean-linkcheck $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." -doctest: +doctest: clean-doctest $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." From 7f36905533a6c4632e386969a8f8a58633a3a2d8 Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:17:19 +0000 Subject: [PATCH 3/9] Requirements (doc): watchdog for automatic rebuilds --- django/requirements-doc.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/django/requirements-doc.txt b/django/requirements-doc.txt index 7b6d9ca9c4..0204f22981 100644 --- a/django/requirements-doc.txt +++ b/django/requirements-doc.txt @@ -5,3 +5,4 @@ Markdown==2.6.9 Pygments==2.2.0 Sphinx==2.4.2 sphinxcontrib-images==0.9.1 +watchdog[watchmedo]==0.10.2 From 914102aa459c12d459dea04b029a575c85a03e50 Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:18:16 +0000 Subject: [PATCH 4/9] Sphinx usage documentation --- sphinx-doc/source/contributing.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sphinx-doc/source/contributing.rst b/sphinx-doc/source/contributing.rst index 19b84f21c2..ef9fe2b049 100644 --- a/sphinx-doc/source/contributing.rst +++ b/sphinx-doc/source/contributing.rst @@ -410,12 +410,13 @@ ReStructured Text. Documentation from commits pushed to the official CATMAID repository are built by `Read the Docs `_ and hosted at `catmaid.org `_. -To build the general documentation from within your pip virtualenv, run:: +CATMAID's documentation can be built in various formats +by navigating to the ``sphinx-doc`` directory and using `make `_. +The default (i.e. ``make`` with no arguments) is HTML, which builds the documentation at sphinx-doc/build/html/index.html. - cd sphinx-doc - make html - -The built documentation is now in ``sphinx-doc/build/html/index.html``. +Every build target can automatically be built when the source files change, +by using ``make watch-``. +Build the HTML docs, watch for changes, and serve the documentation at ``http://localhost:8889`` using ``make serve``. In-Client Documentation ####################### From 16b6998d7f972bea7fc50edee39eaaba274595af Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:25:06 +0000 Subject: [PATCH 5/9] Requirements (dev): flake8 --- django/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/django/requirements-test.txt b/django/requirements-test.txt index 0af5d20010..a2635e1676 100644 --- a/django/requirements-test.txt +++ b/django/requirements-test.txt @@ -3,3 +3,4 @@ django-migration-testcase==0.0.13 selenium==3.141.0 sauceclient==1.0.0 mypy==0.761; platform_python_implementation != 'PyPy' +flake8==3.7.9 From c91503fa8d32a37ffdfa965987df0b0ae7b2d3f2 Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:25:45 +0000 Subject: [PATCH 6/9] Contributing docs: Python linting --- sphinx-doc/source/contributing.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sphinx-doc/source/contributing.rst b/sphinx-doc/source/contributing.rst index ef9fe2b049..7383d8e607 100644 --- a/sphinx-doc/source/contributing.rst +++ b/sphinx-doc/source/contributing.rst @@ -343,6 +343,8 @@ ci.org/catmaid/CATMAID/branches>`_, several automated processes are performed to help verify the correctness and quality of CATMAID: * :doc:`Unit and integration tests for Django backend ` +* Linting (static analysis) of the python code with flake8 +* Type checking of the python code with mypy * Linting (static analysis) of the javascript code with JSHint * Linting of CSS with csslint * Unit tests of javascript code with QUnit @@ -358,6 +360,12 @@ Django tests are run through Django's admin commands:: cd //django/projects ./manage.py test catmaid.tests +flake8 and mypy are installed along with other python development dependencies. +Run them with:: + + flake8 django + mypy django + JSHint can be `installed from NPM or your platform's package manager `_ and should use CATMAID's config settings:: From e2b6685f04ef3592a53fb8ee1566c82296f9a1c8 Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Thu, 26 Mar 2020 19:26:33 +0000 Subject: [PATCH 7/9] Travis: remove explicit flake8 install --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 685f9b47e8..1c8d048ea8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_install: install: # Install requirements for running CATMAID and its unit tests - ./scripts/travis/install_requirements.sh - - pip install coveralls flake8 + - pip install coveralls - npm install before_script: # Set up and start postgres, create database, write config files From a4c69644ff4bdada7dd27e028b7bca656c50c06b Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Mon, 30 Mar 2020 11:16:55 +0100 Subject: [PATCH 8/9] Docs: fix watch in vagrant By default, watchdog looks for file system events. Files changed by the host do not emit events to the guest and vice versa, so the the watcher would not trigger if it was running in the guest and the files were changed by the host. Watchdog has a "debug" polling mode which resolves this. --- sphinx-doc/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx-doc/Makefile b/sphinx-doc/Makefile index 5449ace105..71017939cf 100644 --- a/sphinx-doc/Makefile +++ b/sphinx-doc/Makefile @@ -39,7 +39,7 @@ clean: -rm -rf $(BUILDDIR)/* clean-%: - rm -rf $(BUILDDIR)/$* + rm -rf $(BUILDDIR)/$*/* apidoc: @echo @@ -61,7 +61,7 @@ watch: watch-html watch-%: $(MAKE) $* - watchmedo shell-command --recursive --wait --pattern "*.rst;*.png;*.svg;*.jpg" --command '$(MAKE) $*' source/ + watchmedo shell-command --debug-force-polling --recursive --wait --pattern "*.rst;*.png;*.svg;*.jpg" --command '$(MAKE) $*' source/ html: clean-html $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html From 2508a5be6bfd586bf9ffab7877e9bcd5642ab8ee Mon Sep 17 00:00:00 2001 From: "Chris L. Barnes" Date: Mon, 30 Mar 2020 11:36:28 +0100 Subject: [PATCH 9/9] Vagrant: clarify port forwarding --- sphinx-doc/source/vagrant.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sphinx-doc/source/vagrant.rst b/sphinx-doc/source/vagrant.rst index 8931f80d8f..2b4f98bbe8 100644 --- a/sphinx-doc/source/vagrant.rst +++ b/sphinx-doc/source/vagrant.rst @@ -53,7 +53,7 @@ Once installed, you need only ``vagrant up`` inside the repository to create a f Tools like VScode need to be able to SSH into the container themselves. They get their configuration from your user's SSH configuration (``~/.ssh/config`` on Linux). Copy and paste the output of ``vagrant ssh-config`` (ignoring any warnings from ruby gems) into this file. -The hostname of the container will be ``catmaid-vm``. +The hostname of the container will be ``catmaid-vm``. .. warning:: @@ -71,8 +71,17 @@ In Linux, ``/`` is the root directory, and ``~`` is the home directory of the us * R packages are in ``~/R`` * Data is written to ``~/data`` -The virtual machine also forwards the ports that PostgreSQL and the Django development server listen on, so that you can access them as if they were local to the host. -To avoid clashing with the host, these are forwarded to 5555 and 8888 respectively. +Some guest ports are forwarded to the host machine so that you can access the database, test with the dev server, and look at the generated sphinx docs. + ++-------------------+------------+-----------+---------------------------------------------------+ +| Service | Guest port | Host port | Notes | ++===================+============+===========+===================================================+ +| PostgreSQL | 5432 | 5555 | | ++-------------------+------------+-----------+---------------------------------------------------+ +| Django dev server | 8888 | 8888 | ``django/projects/manage.py runserver [::]:8888`` | ++-------------------+------------+-----------+---------------------------------------------------+ +| Docs server | 8889 | 8889 | ``cd sphinx-doc && make serve`` | ++-------------------+------------+-----------+---------------------------------------------------+ Setup -----