[#40888] Docker image for API, Swagger UI, dev setup

- Dockerfile for API

- Switch to using latest pip-tools to fix error, add api req compilation

jazzband/pip-tools#1307

- API service in dev environment

- Swagger UI service in dev environment (currently using static swagger.yaml - _not_ for production usage)

dev-environment/Dockerfile

@@ -15,7 +15,7 @@ RUN set -ex \
   && apt-get clean && rm -rf "/var/lib/apt/lists/*" "/tmp/*" "/var/tmp/*" "/usr/share/man/??" "/usr/share/man/??_*"
 
 WORKDIR /requirements/python-requirements
-RUN pip install --no-cache-dir pip-tools==5.3.1
+RUN pip install --no-cache-dir pip-tools==5.5.0
 COPY requirements/python-requirements .
 
 # ADD COMMAND FOR COMPILING ALL requirement-*.txt FILES!

dev-environment/api/dev-settings.toml

@@ -0,0 +1,2 @@
+[server]
+token = ""

dev-environment/api/swagger.yml

@@ -0,0 +1,65 @@
+openapi: 3.0.0
+info:
+  title: OS2datascanner scan API
+  description: >
+    An interface to the OS2datascanner scan pipeline.
+  contact:
+    name: Magenta ApS
+    url: https://www.magenta.dk/
+    email: info@magenta.dk
+  license:
+    name: Mozilla Public License, v. 2.0
+    url: https://mozilla.org/MPL/2.0/
+  version: 1.0.0
+servers:
+  -   url: http://127.0.0.1:8000
+components:
+  securitySchemes:
+    bpsk:
+      type: http
+      scheme: bearer
+      bearerFormat: pre-shared key
+paths:
+  /dummy/1:
+    post:
+      operationId: dummy-1
+      summary: Does nothing
+      description: >
+        This operation returns a positive response (for example,
+        for testing that the API server has been started
+        successfully and that authentication has succeeded).
+      responses:
+        "200":
+          description: a dummy success object
+          content:
+            application/jsonl:
+              example: >
+                {"status": "ok"}
+  /run-scan/1:
+    post:
+      operationId: run-scan-1
+      summary: Runs a scan and yields the results
+      description: >
+        d
+      requestBody:
+        description: dummy
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                test:
+                  type: string
+              example:
+                test: test
+      responses:
+        "200":
+          description: a number of pipeline output messages
+          content:
+            application/jsonl:
+              example: >
+                {}
+                {}
+security:
+  -
+    bpsk: []

dev-environment/docker-compose.yml

@@ -9,6 +9,7 @@ services:
              pip-compile requirements-engine.in &&
              pip-compile requirements-admin.in &&
              pip-compile requirements-report.in &&
+             pip-compile requirements-api.in &&
              pip-compile requirements-test.in &&
              pip-compile requirements-lint.in"
 

docker-compose.yml

@@ -161,6 +161,33 @@ services:
     depends_on:
       - queue
 
+  api_server:
+    build:
+      context: .
+      dockerfile: docker/api/Dockerfile
+      target: application
+    command: [
+        "gunicorn",
+        "--config", "/code/docker/gunicorn-settings.py",
+        "--workers", "2", # only two workers in local dev - to save some resources
+        "--reload", # restart workers when code changes
+        "os2datascanner.server.wsgi"
+    ]
+    ports:
+      - "8070:5000"
+    volumes:
+      - ./dev-environment/api/dev-settings.toml:/user-settings.toml
+      - ./src/os2datascanner:/code/src/os2datascanner
+
+  swagger_ui:
+    image: swaggerapi/swagger-ui:v3.41.1
+    environment:
+      - SWAGGER_JSON=/swagger.yml
+    ports:
+      - "8075:8080"
+    volumes:
+      - "./dev-environment/api/swagger.yml:/swagger.yml:ro"
+
   prometheus:
     image: prom/prometheus
     volumes:

docker/api/Dockerfile

@@ -0,0 +1,82 @@
+# Copyright (C) 2021 Magenta ApS, http://magenta.dk.
+# Contact: info@magenta.dk.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+################################################################################
+# Changes to this file requires approval from Labs. Please add a person from   #
+# Labs as required approval to your MR if you have any changes.                #
+################################################################################
+
+FROM python:3.6 AS application
+
+LABEL org.opencontainers.image.title="OS2datascanner - API" \
+      org.opencontainers.image.vendor="Magenta ApS" \
+      org.opencontainers.image.licenses="MPL-2.0" \
+      org.opencontainers.image.url="https://os2datascanner.magenta.dk/" \
+      org.opencontainers.image.documentation="https://os2datascanner.readthedocs.io/en/latest/" \
+      org.opencontainers.image.source="https://github.com/os2datascanner/os2datascanner"
+
+# Force the stdout and stderr streams from python to be unbuffered. See
+# https://docs.python.org/3/using/cmdline.html#cmdoption-u
+ENV PYTHONUNBUFFERED=1 \
+  OS2DS_SERVER_SYSTEM_CONFIG_PATH=/code/docker/docker-settings.toml \
+  OS2DS_SERVER_USER_CONFIG_PATH=/user-settings.toml \
+  PYTHONPATH=/code/src/:$PYTHONPATH
+
+# Install system depedencies
+WORKDIR /code/requirements/sys-requirements
+COPY requirements/sys-requirements/sys-requirements-common.txt \
+     requirements/sys-requirements/sys-requirements-engine.txt \
+     ./
+
+# hadolint ignore=DL3008,SC2046
+RUN set -ex \
+  # Add an image specific group and user.
+  # Note: this is a system user/group, but have
+  # UID/GID above the normal SYS_UID_MAX/SYS_GID_MAX of 999, but also above the
+  # automatic ranges of UID_MAX/GID_MAX used by useradd/groupadd.
+  # Hopefully there will be no conflicts with users of the
+  # host system or users of other docker containers.
+  && groupadd -g 73050 -r os2ds_api\
+  && useradd -u 73050 --no-log-init -r -g os2ds_api os2ds_api \
+  # Install system dependencies from file.
+  && apt-get -y update \
+  && apt-get -y install --no-install-recommends $(grep -oh '^[^#][[:alnum:].-]*' sys-requirements*.txt) \
+  # clean up after apt-get and man-pages
+  && apt-get clean && rm -rf "/var/lib/apt/lists/*" "/tmp/*" "/var/tmp/*" "/usr/share/man/??" "/usr/share/man/??_*"
+
+# Install python requirements
+WORKDIR /code/requirements/python-requirements
+COPY requirements/python-requirements/requirements-api.txt \
+     requirements/python-requirements/requirements-test.txt \
+     requirements/python-requirements/requirements-lint.txt \
+     ./
+# hadolint ignore=DL4006
+RUN find requirements*.txt -print0 | xargs -0 -n1 pip install -r
+
+WORKDIR /code/docker/
+COPY docker/api/docker-settings.toml ./docker-settings.toml
+COPY docker/gunicorn-settings.py ./gunicorn-settings.py
+
+WORKDIR /code/src/os2datascanner
+COPY src/os2datascanner/server ./server/
+COPY src/os2datascanner/engine2 ./engine2/
+COPY src/os2datascanner/utils ./utils
+COPY src/os2datascanner/__init__.py ./
+
+WORKDIR /code
+COPY VERSION ./
+COPY LICENSE ./
+COPY README.rst ./
+COPY NEWS.rst ./
+
+WORKDIR /code/src/os2datascanner/server
+USER os2ds_api:os2ds_api
+EXPOSE 5000
+
+CMD ["gunicorn", \
+     "--config", "/code/docker/gunicorn-settings.py", \
+     "os2datascanner.server.wsgi"]

docker/api/gunicorn-settings.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2020 Magenta ApS, http://magenta.dk.
+# Contact: info@magenta.dk.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+################################################################################
+# Changes to this file requires approval from Labs. Please add a person from   #
+# Labs as required approval to your MR if you have any changes.                #
+################################################################################
+
+
+# Settings for gunicorn in docker.
+import multiprocessing
+
+
+bind = "0.0.0.0:5000"
+workers = multiprocessing.cpu_count() * 2 + 1
+# default directory for heartbeat file is in /tmp, which in some Linux distros is 
+# stored in memory via tmpfs filesystem. Docker containers, however, do not have 
+# /tmp on tmpfs by default - so we use /dev/shm
+# https://pythonspeed.com/articles/gunicorn-in-docker/
+worker_tmp_dir = "/dev/shm" 
+accesslog =  "-"

requirements/README.rst

@@ -3,8 +3,9 @@ requirements and python requirements have been split into several files
 according to the "inheritance" tree below:
 
 ::
-
-              ------------------------- engine
+                                                  ------- api
+                                                /
+              ------------------------- engine -
             /
     common -                    ------- admin
             \                 /

requirements/python-requirements/README.rst

@@ -4,8 +4,9 @@ The ``.in``-files used by ``pip-tools`` are organized according to the "inherita
 tree below, and each ``.in``-file includes all packages from its "parent file":
 
 ::
-
-              ------------------------- engine
+                                                  ------- api
+                                                /
+              ------------------------- engine -
             /
     common -                    ------- admin
             \                 /

requirements/python-requirements/requirements-api.in

@@ -0,0 +1,5 @@
+-r requirements-common.in
+-r requirements-engine.in
+-c requirements-engine.txt
+
+gunicorn