♻️ Maintenance: diverse fixes (codestyle, openapi, dockerignore) (#2593)

* fix missing openapi entries
* ignore mypy cache files in dockerignore
* fix call of codestyle recipe in packages
* director-v2 to restart when packages are updated

Author: sanderegg

.dockerignore

@@ -28,3 +28,6 @@ dask*-space/
 
 # produced when mounting volumes on docker and pip installing
 .local/
+
+# mypy cache
+**/.mypy_cache/

api/specs/webserver/components/schemas/files.yaml

@@ -51,6 +51,8 @@ FileMetaData:
       type: integer
     parent_id:
       type: string
+    entity_tag:
+      type: string
   example:
     file_uuid: "simcore-testing/105/1000/3"
     location_id: "0"
@@ -71,6 +73,7 @@ FileMetaData:
     last_modified: "2019-06-19T12:29:03.78852Z"
     file_size: 73
     parent_id: "N:collection:e263da07-2d89-45a6-8b0f-61061b913873"
+    entity_tag: a87ff679a2f3e71d9181a67b7542122c
 
 FileMetaDataArray:
   type: array

ci/github/unit-testing/dynamic-sidecar.bash

@@ -1,29 +1,32 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash;
-    pushd services/dynamic-sidecar; pip3 install -r requirements/ci.txt -r requirements/_tools.txt; popd;
-    pip list -v
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/dynamic-sidecar
+  pip3 install -r requirements/ci.txt -r requirements/_tools.txt
+  popd
+  pip list -v
 }
 
-codestyle(){
-    scripts/codestyle.bash ci simcore_service_dynamic_sidecar services/dynamic-sidecar 
+codestyle() {
+  pushd services/dynamic-sidecar
+  make codestyle-ci
+  popd
 }
 
 test() {
-    pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/dynamic-sidecar/tests/unit
+  pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/dynamic-sidecar/tests/unit
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

packages/models-library/requirements/ci.txt

@@ -11,8 +11,8 @@
 --requirement _test.txt
 
 # installs this repo's packages
+../postgres-database/[migration]
 ../pytest-simcore/
-../postgres-database/
 
 # current module
 .

packages/models-library/requirements/dev.txt

@@ -12,8 +12,8 @@
 --requirement _tools.txt
 
 # installs this repo's packages
---editable ../pytest-simcore/
 --editable ../postgres-database/[migration]
+--editable ../pytest-simcore/
 
 # current module
 --editable .

scripts/codestyle.bash

@@ -6,12 +6,10 @@ set -o pipefail
 IFS=$'\n\t'
 
 SRC_DIRECTORY_NAME=${2}
-BASE_PATH_DIR=${3-MISSING_DIR}
 
 
 # global config for entire repo
 PYLINT_CONFIG="$(git rev-parse --show-toplevel)/.pylintrc"
-MYPY_CONFIG="$(git rev-parse --show-toplevel)/mypy.ini"
 
 
 # used for development (fails on pylint and mypy)
@@ -24,23 +22,20 @@ development() {
   echo "pylint"
   pylint --rcfile="$PYLINT_CONFIG" src/"$SRC_DIRECTORY_NAME" tests/
   echo "mypy"
-  mypy --ignore-missing-imports --config-file "$MYPY_CONFIG" src/"$SRC_DIRECTORY_NAME" tests/
+  make mypy
 }
 
 # invoked by ci as test (also fails on isort and black)
 ci() {
-  echo "checking codestyle in service=$BASE_PATH_DIR with source_directory=$SRC_DIRECTORY_NAME"
+  echo "enforcing codestyle to source_directory=$SRC_DIRECTORY_NAME"
   echo "isort"
-  isort --check setup.py "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  isort --check setup.py src/"$SRC_DIRECTORY_NAME" tests
   echo "black"
-  black --check "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  black --check src/"$SRC_DIRECTORY_NAME" tests
   echo "pylint ..."
-  pylint --rcfile="$PYLINT_CONFIG" "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  pylint --rcfile="$PYLINT_CONFIG" src/"$SRC_DIRECTORY_NAME" tests/
   echo "mypy ..."
-  # installing all missing stub packages (e.g. types-PyYAML, types-aiofiles, etc)
-  python3 -m pip install types-aiofiles types-PyYAML types-ujson
-  # runs mypy
-  mypy --config-file "$MYPY_CONFIG" --ignore-missing-imports "$BASE_PATH_DIR"/src/"$SRC_DIRECTORY_NAME" "$BASE_PATH_DIR"/tests
+  make --silent mypy
 }
 
 # Allows to call a function based on arguments passed to the script

scripts/common-package.Makefile

@@ -14,7 +14,10 @@
 # NOTE $(CURDIR) in this file refers to the directory where this file is included
 
 # Variable based on conventions (override if they do not apply)
+PACKAGE_NAME      = $(notdir $(CURDIR))
+PY_PACKAGE_NAME  = $(subst -,_,$(PACKAGE_NAME))
 PACKAGE_VERSION  := $(shell cat VERSION)
+SRC_DIR           = $(abspath $(CURDIR)/src/$(PY_PACKAGE_NAME))
 
 export PACKAGE_VERSION
 

scripts/common.Makefile

@@ -133,8 +133,8 @@ code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis
 
 
 .PHONY: codestyle
-codestyle: ## enforces codestyle (isort & black) finally runs pylint & mypy
-	@$(SCRIPTS_DIR)/codestyle.bash development $(shell basename "${SRC_DIR}")
+codestyle codestyle-ci: ## enforces codestyle (isort & black) finally runs pylint & mypy
+	@$(SCRIPTS_DIR)/codestyle.bash $(if $(findstring -ci,$@),ci,development) $(shell basename "${SRC_DIR}")
 
 .PHONY: github-workflow-job
 github-workflow-job: ## runs a github workflow job using act locally, run using "make github-workflow-job job=JOB_NAME"

scripts/mypy.bash

@@ -6,27 +6,35 @@ set -o pipefail
 IFS=$'\n\t'
 
 
-image_name="$(basename $0):latest"
+image_name="$(basename "$0"):latest"
 
 
-docker build --tag "$image_name" -<<EOF
+docker buildx build --tag "$image_name" &>/dev/null -<<EOF
 FROM python:3.8.10-slim-buster
-RUN pip install --upgrade pip && pip install mypy pydantic[email]
+RUN pip install --upgrade pip \
+    && pip install mypy \
+                  pydantic[email] \
+                  types-aiofiles \
+                  types-PyYAML \
+                  types-ujson \
+                  types-setuptools
 ENTRYPOINT ["mypy"]
 EOF
 
 
-target_path=$(realpath ${1:-Please give target path as argument})
+target_path=$(realpath "${1:-Please give target path as argument}")
 cd "$(dirname "$0")"
-default_mypy_config="$(dirname ${PWD})/mypy.ini"
-mypy_config=$(realpath ${2:-${default_mypy_config}})
+default_mypy_config="$(git rev-parse --show-toplevel)/mypy.ini"
+mypy_config=$(realpath "${2:-${default_mypy_config}}")
 
-echo mypying ${target_path} using config in ${mypy_config}...
-
-echo $default_mypy_config
+echo mypying "${target_path}" using config in "${mypy_config}"...
+echo using "$(docker run --rm "$image_name" --version)"
 docker run --rm \
-  -v ${mypy_config}:/config/mypy.ini \
-  -v ${target_path}:/src \
+  --volume /etc/passwd:/etc/passwd:ro \
+  --volume /etc/group:/etc/group:ro \
+  --user $(id -u):$(id -g) \
+  --volume "${mypy_config}":/config/mypy.ini \
+  --volume "${target_path}":/src \
   --workdir=/src \
   "$image_name" \
     --config-file /config/mypy.ini \

services/director-v2/docker/boot.sh

@@ -39,7 +39,8 @@ if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]; then
   exec uvicorn simcore_service_director_v2.main:the_app \
     --reload \
     --host 0.0.0.0 \
-    --reload-dir services/director-v2/src/simcore_service_director_v2
+    --reload-dir services/director-v2/src/simcore_service_director_v2 \
+    --reload-dir packages
 else
   exec uvicorn simcore_service_director_v2.main:the_app \
     --host 0.0.0.0

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/dependencies.py

@@ -7,7 +7,7 @@
 
 
 def get_app_state(request: Request) -> State:
-    return request.app.state  # type: ignore
+    return request.app.state
 
 
 def get_application_health(

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/utils.py

@@ -53,7 +53,7 @@ async def write_to_tmp_file(file_contents: str) -> AsyncGenerator[Path, None]:
     try:
         yield file_path
     finally:
-        await aiofiles.os.remove(file_path)
+        await aiofiles.os.remove(file_path)  # type: ignore
 
 
 @asynccontextmanager

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -5276,6 +5276,8 @@ paths:
                       type: integer
                     parent_id:
                       type: string
+                    entity_tag:
+                      type: string
                   example:
                     file_uuid: simcore-testing/105/1000/3
                     location_id: '0'
@@ -5296,6 +5298,7 @@ paths:
                     last_modified: '2019-06-19T12:29:03.78852Z'
                     file_size: 73
                     parent_id: 'N:collection:e263da07-2d89-45a6-8b0f-61061b913873'
+                    entity_tag: a87ff679a2f3e71d9181a67b7542122c
         default:
           description: Default http error response body
           content:
@@ -5528,6 +5531,8 @@ paths:
                     type: integer
                   parent_id:
                     type: string
+                  entity_tag:
+                    type: string
                 example:
                   file_uuid: simcore-testing/105/1000/3
                   location_id: '0'
@@ -5548,6 +5553,7 @@ paths:
                   last_modified: '2019-06-19T12:29:03.78852Z'
                   file_size: 73
                   parent_id: 'N:collection:e263da07-2d89-45a6-8b0f-61061b913873'
+                  entity_tag: a87ff679a2f3e71d9181a67b7542122c
     patch:
       summary: Update File Metadata
       tags:
@@ -5608,6 +5614,8 @@ paths:
                   type: integer
                 parent_id:
                   type: string
+                entity_tag:
+                  type: string
               example:
                 file_uuid: simcore-testing/105/1000/3
                 location_id: '0'
@@ -5628,6 +5636,7 @@ paths:
                 last_modified: '2019-06-19T12:29:03.78852Z'
                 file_size: 73
                 parent_id: 'N:collection:e263da07-2d89-45a6-8b0f-61061b913873'
+                entity_tag: a87ff679a2f3e71d9181a67b7542122c
       responses:
         '200':
           description: Returns file metadata
@@ -5674,6 +5683,8 @@ paths:
                     type: integer
                   parent_id:
                     type: string
+                  entity_tag:
+                    type: string
                 example:
                   file_uuid: simcore-testing/105/1000/3
                   location_id: '0'
@@ -5694,6 +5705,7 @@ paths:
                   last_modified: '2019-06-19T12:29:03.78852Z'
                   file_size: 73
                   parent_id: 'N:collection:e263da07-2d89-45a6-8b0f-61061b913873'
+                  entity_tag: a87ff679a2f3e71d9181a67b7542122c
   '/storage/locations/{location_id}/datasets/{dataset_id}/metadata':
     get:
       summary: Get Files Metadata
@@ -5759,6 +5771,8 @@ paths:
                       type: integer
                     parent_id:
                       type: string
+                    entity_tag:
+                      type: string
                   example:
                     file_uuid: simcore-testing/105/1000/3
                     location_id: '0'
@@ -5779,6 +5793,7 @@ paths:
                     last_modified: '2019-06-19T12:29:03.78852Z'
                     file_size: 73
                     parent_id: 'N:collection:e263da07-2d89-45a6-8b0f-61061b913873'
+                    entity_tag: a87ff679a2f3e71d9181a67b7542122c
         default:
           description: Default http error response body
           content: