daq-tools · amotl · Jan 17, 2021 · Jan 17, 2021 · Jan 17, 2021 · Jan 17, 2021
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -0,0 +1,103 @@
+name: Tests
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+jobs:
+
+  tests:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ] # , macos-latest, windows-latest ]
+        python-version: [ "3.6", "3.7", "3.8" ]
+        include:
+        - os: ubuntu-latest
+          path: ~/.cache/pip
+        #- os: macos-latest
+        #  path: ~/Library/Caches/pip
+        #- os: windows-latest
+        #  path: ~\AppData\Local\pip\Cache
+
+    # https://docs.github.com/en/free-pro-team@latest/actions/guides/about-service-containers
+    services:
+
+      # TODO: Maybe use https://github.com/marketplace/actions/mosquitto-mqtt-broker-in-github-actions
+      mosquitto:
+        image: eclipse-mosquitto:1.6.12
+        ports:
+          - 1883:1883
+          - 9001:9001
+
+      influxdb:
+        image: influxdb:1.8.3
+        ports:
+          - 8083:8083
+          - 8086:8086
+
+      grafana:
+        image: grafana/grafana:7.3.3
+        ports:
+          - 3000:3000
+        env:
+          GF_SECURITY_ADMIN_PASSWORD: admin
+
+      mongodb:
+        image: mongo:4.4.3
+        ports:
+          - 27017:27017
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+
+    name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }}
+    steps:
+
+      - name: Acquire sources
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Apply caching of dependencies
+        uses: actions/cache@v2
+        with:
+          path: ${{ matrix.path }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements-dev.txt', 'requirements-test.txt', 'requirements-docs.txt') }}
+          restore-keys: |
+           ${{ runner.os }}-pip-
+
+      - name: Install program
+        run: |
+          pip install --requirement=requirements-dev.txt
+          pip install --requirement=requirements-test.txt
+          pip install --editable=.[daq,daq_geospatial,export,scientific,firmware]
+
+      - name: Run tests
+        run: |
+
+          # Run unit- and integration-tests
+          pytest kotori test
+
+          # Run doctests
+          export NOSE_IGNORE_FILES="test_.*\.py"
+          nosetests \
+            --with-doctest --doctest-tests --doctest-extension=rst \
+            kotori/*.py \
+            kotori/daq/{application,graphing,services,storage,strategy} \
+            kotori/daq/intercom/{mqtt/paho.py,udp.py,wamp.py} \
+            kotori/firmware kotori/io kotori/vendor/hiveeyes
+
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
+
+      - name: Build documentation
+        run: |
+          pip install --requirement=requirements-docs.txt
+          SPHINXOPTS="-j auto" make --directory=doc html
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -5,6 +5,8 @@ Changelog
 
 in progress
 ===========
+- Upgrade to Python 3
+
 
 Breaking changes
 ----------------
@@ -248,7 +250,7 @@ Clients
     - Use ptrace.getkotori.org for documentation media assets
     - Correct GitHub repository url
     - Add documentation about Nginx reverse proxy setup
-    - Add installation instructions for Mac OS X
+    - Add installation instructions for macOS
 
 
 

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# (c) 2014-2020 Andreas Motl <andreas.motl@getkotori.org>
+# (c) 2014-2021 Andreas Motl <andreas.motl@getkotori.org>
 
 # ============
 # Main targets
@@ -10,33 +10,27 @@
 # Configuration
 # -------------
 
-$(eval venvpath     := .venv2)
-$(eval pip          := $(venvpath)/bin/pip)
-$(eval python       := $(venvpath)/bin/python)
-$(eval pytest       := $(venvpath)/bin/pytest)
-$(eval bumpversion  := $(venvpath)/bin/bumpversion)
-$(eval twine        := $(venvpath)/bin/twine)
-
-$(eval venv3        := .venv3)
-$(eval pip3         := $(venv3)/bin/pip)
-$(eval python3      := $(venv3)/bin/python)
-$(eval sphinx       := $(venv3)/bin/sphinx-build)
+$(eval venv         := .venv)
+$(eval pip          := $(venv)/bin/pip)
+$(eval python       := $(venv)/bin/python)
+$(eval pytest       := $(venv)/bin/pytest)
+$(eval nosetests    := $(venv)/bin/nosetests)
+$(eval bumpversion  := $(venv)/bin/bumpversion)
+$(eval twine        := $(venv)/bin/twine)
+$(eval sphinx       := $(venv)/bin/sphinx-build)
 
 
 # Setup Python virtualenv
-setup-virtualenv2:
-	@test -e $(python) || `command -v virtualenv` --python=python2 --no-site-packages $(venvpath)
-
-setup-virtualenv3:
-	@test -e $(python3) || `command -v virtualenv` --python=python3 --no-site-packages $(venv3)
+setup-virtualenv:
+	@test -e $(python) || python3 -mvenv $(venv)
 
-virtualenv-docs: setup-virtualenv3
-	@$(pip3) --quiet install --requirement requirements-docs.txt
+virtualenv-docs: setup-virtualenv
+	@$(pip) --quiet install --requirement=requirements-docs.txt
 
-virtualenv-dev: setup-virtualenv2
-	@$(pip) install --upgrade --requirement requirements-dev.txt
-	@$(pip) install --upgrade --requirement requirements-test.txt
-	@$(pip) install --upgrade -e.[daq,daq_geospatial,export,firmware]
+virtualenv-dev: setup-virtualenv
+	@$(pip) install --upgrade --requirement=requirements-dev.txt
+	@$(pip) install --upgrade --requirement=requirements-test.txt
+	@$(pip) install --upgrade --editable=.[daq,daq_geospatial,export,scientific,firmware]
 
 
 # =======
@@ -208,7 +202,7 @@ sdist:
 pypi-upload: install-releasetools
 	twine upload --skip-existing --verbose dist/*.tar.gz
 
-install-releasetools: setup-virtualenv2
+install-releasetools: setup-virtualenv
 	@$(pip) install --quiet --requirement requirements-release.txt --upgrade
 
 
@@ -229,31 +223,31 @@ check-bump-options:
 #
 
 .PHONY: test
-pytest:
+pytest: virtualenv-dev
 
 	# Run pytest.
-	$(venvpath)/bin/pytest kotori test --verbose --log-level DEBUG --log-cli-level DEBUG --log-format='%(asctime)-15s [%(name)-35s] %(levelname)-8s: %(message)s' --log-date-format='%Y-%m-%dT%H:%M:%S%z'
+	$(pytest) kotori test -vvv --log-level DEBUG --log-cli-level DEBUG --log-format='%(asctime)-15s [%(name)-35s] %(levelname)-8s: %(message)s' --log-date-format='%Y-%m-%dT%H:%M:%S%z'
 
-nosetest:
+nosetest: virtualenv-dev
 
 	# Run nosetest.
 	@# https://nose.readthedocs.org/en/latest/plugins/doctests.html
 	@# https://nose.readthedocs.org/en/latest/plugins/cover.html
 	export NOSE_IGNORE_FILES="test_.*\.py"; \
-	$(venvpath)/bin/nosetests --with-doctest --doctest-tests --doctest-extension=rst --verbose \
+	$(nosetests) --with-doctest --doctest-tests --doctest-extension=rst --verbose \
 		kotori/*.py kotori/daq/{application,graphing,services,storage,strategy} kotori/daq/intercom/{mqtt/paho.py,udp.py,wamp.py} kotori/firmware kotori/io kotori/vendor/hiveeyes
 
 test: pytest nosetest
 
-test-coverage:
-	$(venvpath)/bin/nosetests \
+test-coverage: virtualenv-dev
+	$(nosetests) \
 		--with-doctest --doctest-tests --doctest-extension=rst \
 		--with-coverage --cover-package=kotori --cover-tests \
 		--cover-html --cover-html-dir=coverage/html --cover-xml --cover-xml-file=coverage/coverage.xml
 
 docs-html: virtualenv-docs
 	touch doc/source/index.rst
-	export SPHINXBUILD="`pwd`/$(sphinx)"; cd doc; make html
+	SPHINXBUILD="`pwd`/$(sphinx)" SPHINXOPTS="-j auto" make --directory=doc html
 
 
 # ==========================================

diff --git a/README.rst b/README.rst
@@ -33,22 +33,22 @@ Kotori
 
 - **Status**:
 
-  .. image:: https://img.shields.io/badge/Python-2.7-green.svg
-        :target: https://python.org
+  .. image:: https://github.com/daq-tools/kotori/workflows/Tests/badge.svg
+        :target: https://github.com/daq-tools/kotori/actions?workflow=Tests
 
-  .. image:: https://img.shields.io/github/tag/hiveeyes/terkin-datalogger.svg
-        :target: https://github.com/daq-tools/kotori
+  .. image:: https://img.shields.io/pypi/pyversions/kotori.svg
+        :target: https://python.org
 
-  .. image:: https://img.shields.io/pypi/status/kotori.svg
+  .. image:: https://img.shields.io/pypi/v/kotori.svg
         :target: https://pypi.org/project/kotori/
 
-  .. image:: https://img.shields.io/pypi/v/kotori.svg
+  .. image:: https://img.shields.io/pypi/dm/kotori.svg
         :target: https://pypi.org/project/kotori/
 
-  .. image:: https://img.shields.io/pypi/l/kotori.svg
+  .. image:: https://img.shields.io/pypi/status/kotori.svg
         :target: https://pypi.org/project/kotori/
 
-  .. image:: https://img.shields.io/pypi/dm/kotori.svg
+  .. image:: https://img.shields.io/pypi/l/kotori.svg
         :target: https://pypi.org/project/kotori/
 
 

diff --git a/doc/Makefile b/doc/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-#SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = build
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+PAPER         ?=
+BUILDDIR      ?= build
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)

diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -54,7 +54,7 @@
 
 # General information about the project.
 project = u'Kotori'
-copyright = u'2013-2020, The Kotori Developers'
+copyright = u'2013-2021, The Kotori Developers'
 author = u'The Kotori Developers'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -355,17 +355,17 @@ def setup(app):
 
     # foundation
     # Bootstrap conflicts with Sphinx
-    #app.add_stylesheet("assets/css/bootstrap.min.css")
-    app.add_stylesheet("assets/css/font-awesome.min.css")
-    app.add_stylesheet("assets/css/font-entypo.css")
-    app.add_stylesheet("assets/css/hexagons.min.css")
+    #app.add_css_file("assets/css/bootstrap.min.css")
+    app.add_css_file("assets/css/font-awesome.min.css")
+    app.add_css_file("assets/css/font-entypo.css")
+    app.add_css_file("assets/css/hexagons.min.css")
     # jQuery 2.1.0 conflicts with jQuery 1.11.1 from Sphinx
-    #app.add_javascript("assets/js/jquery-2.1.0.min.js")
-    app.add_javascript("assets/js/hexagons.min.js")
+    #app.add_js_file("assets/js/jquery-2.1.0.min.js")
+    app.add_js_file("assets/js/hexagons.min.js")
 
     # application
     #app.add_javascript("custom.js")
-    app.add_stylesheet("css/kotori-sphinx.css")
+    app.add_css_file("css/kotori-sphinx.css")
 
 # Link with BERadio and Hiveeyes projects
 intersphinx_mapping = {

diff --git a/doc/source/development/backlog.rst b/doc/source/development/backlog.rst
@@ -15,6 +15,16 @@ Backlog
 ----
 
 
+****
+2021
+****
+- Unlock Python 3.9 (needs PyTables)
+- Run black and isort
+- Add CI testing on GHA
+- Logrotate configuration for kotori.log
+- Completely replace Bunch with Munch
+
+
 ****
 2020
 ****

diff --git a/doc/source/development/research/acquisition-hardware.rst b/doc/source/development/research/acquisition-hardware.rst
@@ -98,6 +98,9 @@ Miscellaneous
   | DEWESoft produces hardware and software for test measurement, data aquisition, and storage.
   | https://github.com/costerwi/dwdatareader
 
+- | Pythonic access to National Instruments NI DAQmx hardware
+  | https://github.com/slightlynybbled/daqmx
+
 
 
 *****

diff --git a/doc/source/development/tests.rst b/doc/source/development/tests.rst
@@ -2,14 +2,17 @@
 Integration tests
 #################
 
+
 *****
 About
 *****
 The tests are mostly full integration tests. They are testing the whole system
 and the interactions between the subsystems.
 
 The test suite will assume running instances of Mosquitto, Grafana, InfluxDB and
-MongoDB and fire up an in-process instance of Kotori to complement these.
+MongoDB and fire up an in-process instance of Kotori to complement these. Please
+have a look at :ref:`setup-docker` in order to get the complementing services
+up and running in a quick and ad hoc manner.
 
 Then, messages are published to the MQTT bus by shelling out to ``mosquitto_pub``.
 After that, InfluxDB will be checked to contain the right data and Grafana will

diff --git a/doc/source/faq/processing-tags.rst b/doc/source/faq/processing-tags.rst
@@ -88,8 +88,8 @@ Question
 ******
 Answer
 ******
-When installing Kotori from the Debian package,
-it will unfold at ``/opt/kotori/lib/python2.7/site-packages/kotori``.
+When installing Kotori from the Debian package, it will unfold
+at e.g. ``/opt/kotori/lib/python3.8/site-packages/kotori``.
 
 .. attention::
 

diff --git a/doc/source/gallery/firmware.rst b/doc/source/gallery/firmware.rst
@@ -93,7 +93,7 @@ to distinguish different build artifacts from each other.
     How to program the firmware (e.g. using avrdude)?
 
     - Windows: http://m8051.blogspot.de/2015/01/avrdude-on-windows-long-time-after.html
-    - Mac OS X: https://www.pololu.com/docs/0J36/5.c, https://www.obdev.at/products/crosspack/
+    - macOS: https://www.pololu.com/docs/0J36/5.c, https://www.obdev.at/products/crosspack/
 
 
 *******