Add CI on GitHub Actions

Author: amotl

.github/workflows/tests.yml

@@ -0,0 +1,103 @@
+name: Tests
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+jobs:
+
+  tests:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ] # , macos-latest, windows-latest ]
+        python-version: [ "3.6", "3.7", "3.8" ]
+        include:
+        - os: ubuntu-latest
+          path: ~/.cache/pip
+        #- os: macos-latest
+        #  path: ~/Library/Caches/pip
+        #- os: windows-latest
+        #  path: ~\AppData\Local\pip\Cache
+
+    # https://docs.github.com/en/free-pro-team@latest/actions/guides/about-service-containers
+    services:
+
+      # TODO: Maybe use https://github.com/marketplace/actions/mosquitto-mqtt-broker-in-github-actions
+      mosquitto:
+        image: eclipse-mosquitto:1.6.12
+        ports:
+          - 1883:1883
+          - 9001:9001
+
+      influxdb:
+        image: influxdb:1.8.3
+        ports:
+          - 8083:8083
+          - 8086:8086
+
+      grafana:
+        image: grafana/grafana:7.3.3
+        ports:
+          - 3000:3000
+        env:
+          GF_SECURITY_ADMIN_PASSWORD: admin
+
+      mongodb:
+        image: mongo:4.4.3
+        ports:
+          - 27017:27017
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+
+    name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }}
+    steps:
+
+      - name: Acquire sources
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Apply caching of dependencies
+        uses: actions/cache@v2
+        with:
+          path: ${{ matrix.path }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements-dev.txt', 'requirements-test.txt', 'requirements-docs.txt') }}
+          restore-keys: |
+           ${{ runner.os }}-pip-
+
+      - name: Install program
+        run: |
+          pip install --requirement=requirements-dev.txt
+          pip install --requirement=requirements-test.txt
+          pip install --editable=.[daq,daq_geospatial,export,scientific,firmware]
+
+      - name: Run tests
+        run: |
+
+          # Run unit- and integration-tests
+          pytest kotori test
+
+          # Run doctests
+          export NOSE_IGNORE_FILES="test_.*\.py"
+          nosetests \
+            --with-doctest --doctest-tests --doctest-extension=rst \
+            kotori/*.py \
+            kotori/daq/{application,graphing,services,storage,strategy} \
+            kotori/daq/intercom/{mqtt/paho.py,udp.py,wamp.py} \
+            kotori/firmware kotori/io kotori/vendor/hiveeyes
+
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
+
+      - name: Build documentation
+        run: |
+          pip install --requirement=requirements-docs.txt
+          SPHINXOPTS="-j auto" make --directory=doc html

Makefile

@@ -226,7 +226,7 @@ check-bump-options:
 pytest: virtualenv-dev
 
 	# Run pytest.
-	$(pytest) kotori test --verbose --log-level DEBUG --log-cli-level DEBUG --log-format='%(asctime)-15s [%(name)-35s] %(levelname)-8s: %(message)s' --log-date-format='%Y-%m-%dT%H:%M:%S%z'
+	$(pytest) kotori test -vvv --log-level DEBUG --log-cli-level DEBUG --log-format='%(asctime)-15s [%(name)-35s] %(levelname)-8s: %(message)s' --log-date-format='%Y-%m-%dT%H:%M:%S%z'
 
 nosetest: virtualenv-dev
 

README.rst

@@ -33,6 +33,9 @@ Kotori
 
 - **Status**:
 
+  .. image:: https://github.com/daq-tools/kotori/workflows/Tests/badge.svg
+        :target: https://github.com/daq-tools/kotori/actions?workflow=Tests
+
   .. image:: https://img.shields.io/pypi/pyversions/kotori.svg
         :target: https://python.org
 

setup.py

@@ -76,7 +76,7 @@
     # Data export: Basic formats
     'export': [
         'pyinfluxql==0.0.1',
-        'pandas>=1.2.0,<1.3',
+        'pandas>=1.1.5,<1.3',
         'numpy>=1.19.2,<1.20',
         'XlsxWriter>=1.3.6,<1.4',
     ],

test/util.py

@@ -5,6 +5,7 @@
 import logging
 import random
 import string
+import sys
 
 import pytest
 import requests
@@ -25,9 +26,9 @@ def boot_kotori(config):
     options = {
         '--config': config,
         '--debug': True,
-        '--debug-io': True,
-        '--debug-mqtt': True,
-        '--debug-influx': True,
+        '--debug_io': True,
+        '--debug_mqtt': True,
+        '--debug_influx': True,
     }
     loader = kotori.boot(options)
     return loader
@@ -145,10 +146,26 @@ def mqtt_json_sensor(topic, data):
 
 
 def mqtt_sensor(topic, payload):
+
     logger.info('MQTT: Submitting reading')
-    command = "mosquitto_pub -h localhost -t '{topic}' -m '{payload}'".format(topic=topic, payload=payload)
+
+    # When running on CI (GHA), run ``mosquitto_pub`` from Docker image.
+    # https://stackoverflow.com/questions/24319662
+    if os.environ.get("CI"):
+        if sys.platform == "linux":
+            mosquitto_pub = "docker run --rm --network=host eclipse-mosquitto:1.6.12 mosquitto_pub -h localhost"
+        elif sys.platform == "darwin":
+            mosquitto_pub = "docker run --rm eclipse-mosquitto:1.6.12 mosquitto_pub -h host.docker.internal"
+        else:
+            raise NotImplementedError(f"Invoking 'mosquitto_pub' through Docker on '{sys.platform}' not supported yet")
+    else:
+        mosquitto_pub = "mosquitto_pub -h localhost"
+    command = f"{mosquitto_pub} -t '{topic}' -m '{payload}'"
+
     logger.info('Running command {}'.format(command))
-    os.system(command)
+    exitcode = os.system(command)
+    if exitcode != 0:
+        raise ChildProcessError(f"Invoking command failed: {command}. Exit code: {exitcode}")
 
 
 def http_json_sensor(topic, data):