bump weaver 4.5.0 and relevant mongodb changes

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.17.6
+current_version = 1.18.0
 commit = True
 tag = False
 tag_name = {new_version}

CHANGES.md

@@ -16,6 +16,33 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[1.18.0](https://github.com/bird-house/birdhouse-deploy/tree/1.18.0) (2021-12-08)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Upgrade default `Weaver` version to [4.5.0](https://github.com/crim-ca/weaver/blob/master/CHANGES.rst#450-2021-11-25) 
+  (from [4.2.1](https://github.com/crim-ca/weaver/blob/master/CHANGES.rst#421-2021-10-20)) for new features and fixes.
+  Most notable changes are: 
+  - Adds support of `X-WPS-Output-Context` header to define the WPS output nested directory (for user context).
+  - Adds support of `X-Auth-Docker` header to define a private Docker registry authentication token when the 
+    referenced Docker image in the deployed Application Package requires it to fetch it for Process execution. 
+  - Require `MongoDB==5.0` Docker image for Weaver's database.
+  - Fixes related to handling `dismiss` operation of job executions and retrieval of their results.
+  - Fixes related to fetching remote files and propagation of intermediate results between Workflow steps.
+
+## Important
+Because of the new `MongoDB==5.0` database requirement for Weaver that uses (potentially) distinct version from other 
+birds (notably `phoenix` with `MongoDB==3.4`), a separate Docker image is employed only for Weaver. If some processes, 
+jobs, or other Weaver-related data was already defined on one of your server instances, manual transfer between the 
+generic `${DATA_PERSIST_ROOT}/mongodb_persist` to new  `${DATA_PERSIST_ROOT}/mongodb_weaver_persist` directory must 
+be accomplished. The data in the new directory should then be migrated to the new version following the procedure 
+described in [Database Migration](https://pavics-weaver.readthedocs.io/en/latest/installation.html?#database-migration).
+
+## Legal Notice
+While migrating from ``MongoDB==3.4`` to ``MongoDB==5.0``, its license changes from AGPL to SSPL
+(reference: [mongodb/mongo@6ea81c8/README#L89-L95](https://github.com/mongodb/mongo/blob/6ea81c88/README#L89-L95)).
+This should not impact users using the platform for public and Open Source uses, but should be considered otherwise.
+
 [1.17.6](https://github.com/bird-house/birdhouse-deploy/tree/1.17.6) (2021-12-03)
 ------------------------------------------------------------------------------------------------------------------
 

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.17.6.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.18.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.17.6...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.18.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.17.6-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.18.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.17.6
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.18.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

birdhouse/components/weaver/config/weaver/weaver.ini.template

@@ -18,7 +18,7 @@ pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
 # mongodb
-mongodb.host = mongodb
+mongodb.host = weaver-mongodb
 mongodb.port = 27017
 mongodb.db_name = weaver
 

birdhouse/components/weaver/default.env

@@ -30,7 +30,11 @@ VARS="$VARS $EXTRA_VARS"
 export WEAVER_CONFIG=HYBRID
 
 # default release version that will be used to fetch docker images (API mananger & celery workers services)
-export WEAVER_VERSION=4.2.1
+export WEAVER_VERSION=4.5.0
+
+# default release of the MongoDB version employed by Weaver
+# note that MongoDB>=5.0 is REQUIRED for Weaver>=4.5.0
+export WEAVER_MONGODB_VERSION=5.0.4
 
 # real names of the weaver/worker services
 # 'WEAVER_MANAGER_NAME' value will generate "<server-fqdn>/<name>" URI to access its API behind secured proxy

birdhouse/components/weaver/docker-compose-extra.yml

@@ -47,12 +47,12 @@ services:
     #env_file:
     #  - ./config/mongodb/credentials.env
     depends_on:
-      - mongodb
+      - weaver-mongodb
     links:
-      - mongodb
+      - weaver-mongodb
     networks:
       - default
-      - mongodb
+      - weaver-mongodb
     volumes:
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
       - ./components/weaver/config/weaver/data_sources.yml:/opt/local/src/weaver/config/data_sources.yml:ro
@@ -74,17 +74,17 @@ services:
     #  - ./config/mongodb/credentials.env
     depends_on:
       # no 'default' network here such that the worker is not accessible from proxy/http
-      - mongodb
+      - weaver-mongodb
       - weaver  # if not started first, sometimes celery misbehaves and will not pick jobs in queue
       - docker-proxy
     links:
-      - mongodb
+      - weaver-mongodb
     networks:
       # WARNING: *DO NOT* provide 'default' network
       #   Not providing 'default' network ensure that only connection with other images (birds, proxy) can be
       #   accomplished through Weaver API which cannot run sibling dockers by itself. Isolate Dockers execution.
       - docker-proxy
-      - mongodb
+      - weaver-mongodb
     volumes:
       #- wps_private:/data/wps_private
       - ${DATA_PERSIST_ROOT}/wps_private:/data/wps_private
@@ -114,6 +114,19 @@ services:
       timeout: 10s
       retries: 2
 
+  # Dedicated database for Weaver, since other 'mongodb' image does not employ the same version.
+  weaver-mongodb:
+    image: mongo:${WEAVER_MONGODB_VERSION}
+    container_name: weaver-mongodb
+    networks:
+      - weaver-mongodb
+    volumes:
+      - ${DATA_PERSIST_ROOT}/mongodb_weaver_persist:/data/db
+    # Mongodb crash with permission denied errors if the command is not overridden like this
+    command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod'
+    restart: always
+    logging: *default-logging
+
   # Image 'docker-proxy' is the connector between docker image 'worker' jobs and their sibling docker executions
   docker-proxy:
     # references
@@ -150,6 +163,6 @@ services:
     #    constraints: [node.role == manager]
 
 networks:
-  mongodb: {}
+  weaver-mongodb: {}
   docker-proxy:
     driver: bridge