diff --git a/dev-environment/Dockerfile b/dev-environment/Dockerfile index 65bb8872b..612cda003 100644 --- a/dev-environment/Dockerfile +++ b/dev-environment/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex \ && apt-get clean && rm -rf "/var/lib/apt/lists/*" "/tmp/*" "/var/tmp/*" "/usr/share/man/??" "/usr/share/man/??_*" WORKDIR /requirements/python-requirements -RUN pip install --no-cache-dir pip-tools==5.3.1 +RUN pip install --no-cache-dir pip-tools==5.5.0 COPY requirements/python-requirements . # ADD COMMAND FOR COMPILING ALL requirement-*.txt FILES! diff --git a/dev-environment/api/dev-settings.toml b/dev-environment/api/dev-settings.toml new file mode 100644 index 000000000..1af611daa --- /dev/null +++ b/dev-environment/api/dev-settings.toml @@ -0,0 +1,2 @@ +[server] +token = "" diff --git a/dev-environment/api/swagger.yml b/dev-environment/api/swagger.yml new file mode 100644 index 000000000..88b37e3c4 --- /dev/null +++ b/dev-environment/api/swagger.yml @@ -0,0 +1,65 @@ +openapi: 3.0.0 +info: + title: OS2datascanner scan API + description: > + An interface to the OS2datascanner scan pipeline. + contact: + name: Magenta ApS + url: https://www.magenta.dk/ + email: info@magenta.dk + license: + name: Mozilla Public License, v. 2.0 + url: https://mozilla.org/MPL/2.0/ + version: 1.0.0 +servers: + - url: http://127.0.0.1:8000 +components: + securitySchemes: + bpsk: + type: http + scheme: bearer + bearerFormat: pre-shared key +paths: + /dummy/1: + post: + operationId: dummy-1 + summary: Does nothing + description: > + This operation returns a positive response (for example, + for testing that the API server has been started + successfully and that authentication has succeeded). + responses: + "200": + description: a dummy success object + content: + application/jsonl: + example: > + {"status": "ok"} + /run-scan/1: + post: + operationId: run-scan-1 + summary: Runs a scan and yields the results + description: > + d + requestBody: + description: dummy + content: + application/json: + schema: + type: object + properties: + test: + type: string + example: + test: test + responses: + "200": + description: a number of pipeline output messages + content: + application/jsonl: + example: > + {} + {} +security: + - + bpsk: [] diff --git a/dev-environment/docker-compose.yml b/dev-environment/docker-compose.yml index e27c056e6..a45126c78 100644 --- a/dev-environment/docker-compose.yml +++ b/dev-environment/docker-compose.yml @@ -9,6 +9,7 @@ services: pip-compile requirements-engine.in && pip-compile requirements-admin.in && pip-compile requirements-report.in && + pip-compile requirements-api.in && pip-compile requirements-test.in && pip-compile requirements-lint.in" diff --git a/docker-compose.yml b/docker-compose.yml index 6d7c75825..56812fbed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -161,6 +161,33 @@ services: depends_on: - queue + api_server: + build: + context: . + dockerfile: docker/api/Dockerfile + target: application + command: [ + "gunicorn", + "--config", "/code/docker/gunicorn-settings.py", + "--workers", "2", # only two workers in local dev - to save some resources + "--reload", # restart workers when code changes + "os2datascanner.server.wsgi" + ] + ports: + - "8070:5000" + volumes: + - ./dev-environment/api/dev-settings.toml:/user-settings.toml + - ./src/os2datascanner:/code/src/os2datascanner + + swagger_ui: + image: swaggerapi/swagger-ui:v3.41.1 + environment: + - SWAGGER_JSON=/swagger.yml + ports: + - "8075:8080" + volumes: + - "./dev-environment/api/swagger.yml:/swagger.yml:ro" + prometheus: image: prom/prometheus volumes: diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile new file mode 100644 index 000000000..1a54cc17e --- /dev/null +++ b/docker/api/Dockerfile @@ -0,0 +1,82 @@ +# Copyright (C) 2021 Magenta ApS, http://magenta.dk. +# Contact: info@magenta.dk. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +################################################################################ +# Changes to this file requires approval from Labs. Please add a person from # +# Labs as required approval to your MR if you have any changes. # +################################################################################ + +FROM python:3.6 AS application + +LABEL org.opencontainers.image.title="OS2datascanner - API" \ + org.opencontainers.image.vendor="Magenta ApS" \ + org.opencontainers.image.licenses="MPL-2.0" \ + org.opencontainers.image.url="https://os2datascanner.magenta.dk/" \ + org.opencontainers.image.documentation="https://os2datascanner.readthedocs.io/en/latest/" \ + org.opencontainers.image.source="https://github.com/os2datascanner/os2datascanner" + +# Force the stdout and stderr streams from python to be unbuffered. See +# https://docs.python.org/3/using/cmdline.html#cmdoption-u +ENV PYTHONUNBUFFERED=1 \ + OS2DS_SERVER_SYSTEM_CONFIG_PATH=/code/docker/docker-settings.toml \ + OS2DS_SERVER_USER_CONFIG_PATH=/user-settings.toml \ + PYTHONPATH=/code/src/:$PYTHONPATH + +# Install system depedencies +WORKDIR /code/requirements/sys-requirements +COPY requirements/sys-requirements/sys-requirements-common.txt \ + requirements/sys-requirements/sys-requirements-engine.txt \ + ./ + +# hadolint ignore=DL3008,SC2046 +RUN set -ex \ + # Add an image specific group and user. + # Note: this is a system user/group, but have + # UID/GID above the normal SYS_UID_MAX/SYS_GID_MAX of 999, but also above the + # automatic ranges of UID_MAX/GID_MAX used by useradd/groupadd. + # Hopefully there will be no conflicts with users of the + # host system or users of other docker containers. + && groupadd -g 73050 -r os2ds_api\ + && useradd -u 73050 --no-log-init -r -g os2ds_api os2ds_api \ + # Install system dependencies from file. + && apt-get -y update \ + && apt-get -y install --no-install-recommends $(grep -oh '^[^#][[:alnum:].-]*' sys-requirements*.txt) \ + # clean up after apt-get and man-pages + && apt-get clean && rm -rf "/var/lib/apt/lists/*" "/tmp/*" "/var/tmp/*" "/usr/share/man/??" "/usr/share/man/??_*" + +# Install python requirements +WORKDIR /code/requirements/python-requirements +COPY requirements/python-requirements/requirements-api.txt \ + requirements/python-requirements/requirements-test.txt \ + requirements/python-requirements/requirements-lint.txt \ + ./ +# hadolint ignore=DL4006 +RUN find requirements*.txt -print0 | xargs -0 -n1 pip install -r + +WORKDIR /code/docker/ +COPY docker/api/docker-settings.toml ./docker-settings.toml +COPY docker/gunicorn-settings.py ./gunicorn-settings.py + +WORKDIR /code/src/os2datascanner +COPY src/os2datascanner/server ./server/ +COPY src/os2datascanner/engine2 ./engine2/ +COPY src/os2datascanner/utils ./utils +COPY src/os2datascanner/__init__.py ./ + +WORKDIR /code +COPY VERSION ./ +COPY LICENSE ./ +COPY README.rst ./ +COPY NEWS.rst ./ + +WORKDIR /code/src/os2datascanner/server +USER os2ds_api:os2ds_api +EXPOSE 5000 + +CMD ["gunicorn", \ + "--config", "/code/docker/gunicorn-settings.py", \ + "os2datascanner.server.wsgi"] diff --git a/docker/api/docker-settings.toml b/docker/api/docker-settings.toml new file mode 100644 index 000000000..e69de29bb diff --git a/docker/api/gunicorn-settings.py b/docker/api/gunicorn-settings.py new file mode 100644 index 000000000..4a46d5eba --- /dev/null +++ b/docker/api/gunicorn-settings.py @@ -0,0 +1,25 @@ +# Copyright (C) 2020 Magenta ApS, http://magenta.dk. +# Contact: info@magenta.dk. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +################################################################################ +# Changes to this file requires approval from Labs. Please add a person from # +# Labs as required approval to your MR if you have any changes. # +################################################################################ + + +# Settings for gunicorn in docker. +import multiprocessing + + +bind = "0.0.0.0:5000" +workers = multiprocessing.cpu_count() * 2 + 1 +# default directory for heartbeat file is in /tmp, which in some Linux distros is +# stored in memory via tmpfs filesystem. Docker containers, however, do not have +# /tmp on tmpfs by default - so we use /dev/shm +# https://pythonspeed.com/articles/gunicorn-in-docker/ +worker_tmp_dir = "/dev/shm" +accesslog = "-" diff --git a/requirements/README.rst b/requirements/README.rst index 4ca5d396c..91eca8a13 100644 --- a/requirements/README.rst +++ b/requirements/README.rst @@ -3,8 +3,9 @@ requirements and python requirements have been split into several files according to the "inheritance" tree below: :: - - ------------------------- engine + ------- api + / + ------------------------- engine - / common - ------- admin \ / diff --git a/requirements/python-requirements/README.rst b/requirements/python-requirements/README.rst index 630c2d61d..13914b0a8 100644 --- a/requirements/python-requirements/README.rst +++ b/requirements/python-requirements/README.rst @@ -4,8 +4,9 @@ The ``.in``-files used by ``pip-tools`` are organized according to the "inherita tree below, and each ``.in``-file includes all packages from its "parent file": :: - - ------------------------- engine + ------- api + / + ------------------------- engine - / common - ------- admin \ / diff --git a/requirements/python-requirements/requirements-api.in b/requirements/python-requirements/requirements-api.in new file mode 100644 index 000000000..da62eb49f --- /dev/null +++ b/requirements/python-requirements/requirements-api.in @@ -0,0 +1,5 @@ +-r requirements-common.in +-r requirements-engine.in +-c requirements-engine.txt + +gunicorn diff --git a/requirements/python-requirements/requirements-api.txt b/requirements/python-requirements/requirements-api.txt new file mode 100644 index 000000000..9d296611b --- /dev/null +++ b/requirements/python-requirements/requirements-api.txt @@ -0,0 +1,255 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements-api.in +# +beautifulsoup4==4.9.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +cached-property==1.5.1 + # via + # -c requirements-engine.txt + # exchangelib +cachetools==4.1.1 + # via + # -c requirements-engine.txt + # google-auth +certifi==2020.4.5.1 + # via + # -c requirements-engine.txt + # requests +cffi==1.14.0 + # via + # -c requirements-engine.txt + # cryptography + # xattr +chardet==3.0.4 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + # requests +colorama==0.4.3 + # via + # -c requirements-engine.txt + # -r requirements-common.in +cryptography==2.9.2 + # via + # -c requirements-engine.txt + # requests-ntlm +defusedxml==0.6.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + # exchangelib +dnspython==1.16.0 + # via + # -c requirements-engine.txt + # exchangelib +dropbox==10.3.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +exchangelib==3.1.1 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +google-api-core==1.22.2 + # via + # -c requirements-engine.txt + # google-api-python-client +google-api-python-client==1.12.1 + # via + # -c requirements-engine.txt + # -r requirements-common.in +google-auth-httplib2==0.0.4 + # via + # -c requirements-engine.txt + # google-api-python-client +google-auth==1.21.2 + # via + # -c requirements-engine.txt + # google-api-core + # google-api-python-client + # google-auth-httplib2 +googleapis-common-protos==1.52.0 + # via + # -c requirements-engine.txt + # google-api-core +gunicorn==20.0.4 + # via -r requirements-api.in +httplib2==0.18.1 + # via + # -c requirements-engine.txt + # google-api-python-client + # google-auth-httplib2 + # oauth2client +idna==2.9 + # via + # -c requirements-engine.txt + # requests +isodate==0.6.0 + # via + # -c requirements-engine.txt + # exchangelib +lxml==4.5.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + # exchangelib +ntlm-auth==1.4.0 + # via + # -c requirements-engine.txt + # requests-ntlm +oauth2client==4.1.3 + # via + # -c requirements-engine.txt + # -r requirements-common.in +oauthlib==3.1.0 + # via + # -c requirements-engine.txt + # exchangelib + # requests-oauthlib +olefile==0.46 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +pdfrw==0.4 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +pika==1.1.0 + # via + # -c requirements-engine.txt + # -r requirements-common.in +pillow==7.1.1 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +prometheus_client==0.7.1 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +protobuf==3.13.0 + # via + # -c requirements-engine.txt + # google-api-core + # googleapis-common-protos +pyasn1-modules==0.2.8 + # via + # -c requirements-engine.txt + # google-auth + # oauth2client +pyasn1==0.4.8 + # via + # -c requirements-engine.txt + # oauth2client + # pyasn1-modules + # rsa +pycparser==2.20 + # via + # -c requirements-engine.txt + # cffi +pygments==2.6.1 + # via + # -c requirements-engine.txt + # exchangelib +pypdf2==1.26.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +pysmbc==1.0.21 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +python-dateutil==2.8.1 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + # exchangelib +python-magic==0.4.18 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +pytz==2019.3 + # via + # -c requirements-engine.txt + # exchangelib + # google-api-core + # tzlocal +regex==2020.4.4 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +requests-ntlm==1.1.0 + # via + # -c requirements-engine.txt + # exchangelib +requests-oauthlib==1.3.0 + # via + # -c requirements-engine.txt + # exchangelib +requests==2.23.0 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + # dropbox + # exchangelib + # google-api-core + # requests-ntlm + # requests-oauthlib +rsa==4.6 + # via + # -c requirements-engine.txt + # google-auth + # oauth2client +six==1.14.0 + # via + # -c requirements-engine.txt + # cryptography + # dropbox + # google-api-core + # google-api-python-client + # google-auth + # google-auth-httplib2 + # isodate + # oauth2client + # protobuf + # python-dateutil + # structlog +soupsieve==2.0 + # via + # -c requirements-engine.txt + # beautifulsoup4 +structlog==20.1.0 + # via + # -c requirements-engine.txt + # -r requirements-common.in +systemd-python==234 + # via + # -c requirements-engine.txt + # -r requirements-engine.in +toml==0.10.1 + # via + # -c requirements-engine.txt + # -r requirements-common.in +tzlocal==2.0.0 + # via + # -c requirements-engine.txt + # exchangelib +uritemplate==3.0.1 + # via + # -c requirements-engine.txt + # google-api-python-client +urllib3==1.25.9 + # via + # -c requirements-engine.txt + # requests +xattr==0.9.7 + # via + # -c requirements-engine.txt + # -r requirements-engine.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools