Feature/dac 300 weaver compose

.gitignore

@@ -8,7 +8,6 @@ ubuntu-bionic-18.04-cloudimg-console.log
 .idea
 .vscode
 *.bak
-weaver/
 
 # vim temp files
 *.swp

birdhouse/components/README.rst

@@ -61,7 +61,7 @@ How to Enable the Component
   - Add "./components/scheduler" to ``EXTRA_CONF_DIRS``.
   - Set ``AUTODEPLOY_EXTRA_REPOS``, ``AUTODEPLOY_DEPLOY_KEY_ROOT_DIR``,
     ``AUTODEPLOY_PLATFORM_FREQUENCY``, ``AUTODEPLOY_NOTEBOOK_FREQUENCY`` as desired,
-    full documentation in env.local.example_.
+    full documentation in `env.local.example`_.
   - Run once fix-write-perm_ (:download:`download <../deployment/fix-write-perm>`), see doc in script.
 
 
@@ -155,7 +155,7 @@ Usage
 How to Enable the Component
 ---------------------------
 
-- Edit ``env.local`` (a copy of env.local.example_ (:download:`download <../env.local.example>`))
+- Edit ``env.local`` (a copy of `env.local.example`_ (:download:`download <../env.local.example>`))
 
   - Add "./components/monitoring" to ``EXTRA_CONF_DIRS``
   - Set ``GRAFANA_ADMIN_PASSWORD`` to login to Grafana
@@ -229,8 +229,71 @@ Customizing the Component
   Slack or other services accepting webhooks), ``ALERTMANAGER_EXTRA_RECEIVERS``.
 
 
+Weaver
+======
+
+By enabling this component, the `Weaver`_ service will be integrated into the stack.
+
+This component offers `OGC API - Processes`_ interace to WPS components (a.k.a `WPS-REST bindings` and
+`WPS-T (Transactional)` support).
+This provides a RESTful JSON interface with asynchronous WPS processes execution over remote instances. 
+Other WPS components of the birdhouse stack (`finch`_, `flyingpigeon`_, etc.) will also all be registered
+under `Weaver`_ in order to provide a common endpoint to retrieve all available processes, and dispatch 
+their execution to the corresponding service.
+Finally, `Weaver`_ also adds `Docker` image execution capabilities as a WPS process, allowing deployment 
+and execution of custom applications and workflows.
+
+
+Usage
+-----
+
+Once this component is enabled, `Weaver`_ will be accessible at ``https://<PAVICS_FQDN_PUBLIC>/weaver`` endpoint,
+where ``PAVICS_FQDN_PUBLIC`` is defined in your ``env.local`` file.
+
+Full process listing (across WPS providers) should be available using request:
+
+.. code-block::
+
+    GET https://<PAVICS_FQDN_PUBLIC>/weaver/processes?providers=true
+
+Please refer to the `Weaver OpenAPI`_ for complete description of available requests.
+This description will also be accessible via ``https://<PAVICS_FQDN_PUBLIC>/weaver/api`` once the instance is started.
+
+For any specific details about `Weaver`_ configuration parameters, functionalities or questions, please refer to its
+`documentation <https://pavics-weaver.readthedocs.io/en/latest/>`_.
+
+How to Enable the Component
+---------------------------
+
+- Edit ``env.local`` (a copy of `env.local.example`_)
+
+  - Add ``"./components/weaver"`` to ``EXTRA_CONF_DIRS``.
+
+  - Component ``birdhouse/optional-components/all-public-access`` should also be enabled to ensure that `Weaver`_
+    can request ``GetCapabilities`` of every WPS provider to be registered. Publicly inaccessible services will not
+    succeed registration and will not provide the WPS-REST interface.
+
+
+Customizing the Component
+-------------------------
+
+- Edit ``env.local`` (a copy of `env.local.example`_)
+
+  - Optionally, set any additional environment variable overrides amongst values defined in `weaver/default.env`_.
+
+  - Optionally, mount any additional `Weaver`_-specific configuration files
+    (see contents of ``birdhouse/components/weaver/config/weaver``) if extended functionalities need to be defined.
+    Further ``docker-compose-extra.yml`` could be needed to define
+    any other ``volumes`` entries where these component would need to be mounted to.
+
 
 
+.. _finch: https://github.com/bird-house/finch
+.. _flyingpigeon: https://github.com/bird-house/flyingpigeon
+.. _Weaver: https://github.com/crim-ca/weaver
+.. _Weaver OpenAPI: https://pavics-weaver.readthedocs.io/en/latest/api.html
+.. _weaver/default.env: ./weaver/default.env
+.. _OGC API - Processes: https://github.com/opengeospatial/ogcapi-processes
 .. _env.local.example: ../env.local.example
 .. _fix-write-perm: ../deployment/fix-write-perm
 .. _deploy.sh: ../deployment/deploy.sh

birdhouse/components/monitoring/docker-compose-extra.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: "3.4"
 
 services:
   # https://github.com/google/cadvisor/blob/master/docs/running.md

birdhouse/components/scheduler/docker-compose-extra.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: "3.4"
 
 services:
   scheduler:

birdhouse/components/weaver/.gitignore

@@ -0,0 +1,6 @@
+conf.extra-service.d/weaver.conf
+config/magpie/config.yml
+config/weaver/data_sources.json
+config/weaver/request_options.yml
+config/weaver/wps_processes.yml
+config/weaver/weaver.ini

birdhouse/components/weaver/conf.extra-service.d/weaver.conf

@@ -0,0 +1,25 @@
+
+    # NOTE:
+    #   Redirect to internal network of twitcher with Weaver root endpoint and alias allows to set
+    #   the same 'magpie' permissions on the 'weaver' service defined by "WEAVER_MANAGER_NAME".
+    #   This allows verification of the same service user/group permissions references regardless
+    #   whether the *shortcut* Weaver endpoint, the alias or the explicit 'twitcher' proxy route is used.
+    # redirect EMS/ADES to actual secured Weaver path
+    #location /EMS {
+    #    return 302 https://localhost/twitcher/ows/proxy/weaver;
+    #}
+
+    location /weaver {
+        proxy_pass https://localhost/twitcher/ows/proxy/weaver;
+        proxy_set_header Host $host;
+        proxy_buffering off;
+        include /etc/nginx/conf.d/cors.include;
+    }
+
+    # NOTE:
+    #   this is needed only if not using the location already provided by the core configuration
+    #   see 'birdhouse/config/proxy/conf.d/all-services.include.template'
+    # location where process job outputs will be accessible
+    #location ^~ /wpsoutputs/weaver/ {
+    #    alias /data/daccs/wps_outputs/weaver/;
+    #}

birdhouse/components/weaver/conf.extra-service.d/weaver.conf.template

@@ -0,0 +1,25 @@
+
+    # NOTE:
+    #   Redirect to internal network of twitcher with Weaver root endpoint and alias allows to set
+    #   the same 'magpie' permissions on the 'weaver' service defined by "WEAVER_MANAGER_NAME".
+    #   This allows verification of the same service user/group permissions references regardless
+    #   whether the *shortcut* Weaver endpoint, the alias or the explicit 'twitcher' proxy route is used.
+    # redirect EMS/ADES to actual secured Weaver path
+    #location /${WEAVER_CONFIG} {
+    #    return 302 https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME};
+    #}
+
+    location /${WEAVER_MANAGER_NAME} {
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WEAVER_MANAGER_NAME};
+        proxy_set_header Host $host;
+        proxy_buffering off;
+        include /etc/nginx/conf.d/cors.include;
+    }
+
+    # NOTE:
+    #   this is needed only if not using the location already provided by the core configuration
+    #   see 'birdhouse/config/proxy/conf.d/all-services.include.template'
+    # location where process job outputs will be accessible
+    #location ^~ ${WEAVER_WPS_OUTPUTS_PATH}/ {
+    #    alias ${WEAVER_WPS_OUTPUTS_DIR}/;
+    #}

birdhouse/components/weaver/config/magpie/config.yml

@@ -0,0 +1,106 @@
+## see reference example for details:
+# https://github.com/Ouranosinc/Magpie/blob/master/config/config.yml
+
+providers:
+  # definition of Weaver service
+  weaver:
+    url: http://localhost:4001
+    title: Weaver (EMS)
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+
+  # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented, see 'default.env'
+  weaver-wps:
+    url: http://localhost:4001/wps
+    title: Weaver (WPS)
+    public: true
+    c4i: false
+    type: wps
+    sync_type: wps
+
+permissions:
+  # Entrypoint
+  - service: weaver
+    permission: read-match
+    group: anonymous
+    action: create
+
+  # OpenAPI JSON schema
+  - service: weaver
+    resource: /json
+    permission: read
+    group: anonymous
+    action: create
+
+  # OpenAPI Swagger
+  - service: weaver
+    resource: /api
+    permission: read
+    group: anonymous
+    action: create
+
+  # Process deploy/listing (builtin/deployed processes)
+  - service: weaver
+    resource: /processes    # GET is processes listing, POST is deploy: only allow view by anonymous
+    permission: read        # under '/processes/...', JSON 'DescribeProcess', POST job submit, GET results, etc.
+    group: anonymous
+    action: create
+
+  # Providers listing
+  - service: weaver
+    resource: /providers
+    permission: read-match
+    group: anonymous
+    action: create
+
+  # Provider processes listing
+  # (each provider in './components/weaver/config/weaver/wps_processes.yml.template')
+  # catalog
+  - service: weaver
+    resource: /providers/catalog
+    permission: read
+    group: anonymous
+    action: create
+  # finch
+  - service: weaver
+    resource: /providers/finch
+    permission: read
+    group: anonymous
+    action: create
+  # flyingpigeon
+  - service: weaver
+    resource: /providers/flyingpigeon
+    permission: read
+    group: anonymous
+    action: create
+  # hummingbird
+  - service: weaver
+    resource: /providers/hummingbird
+    permission: read
+    group: anonymous
+    action: create
+  # malleefowl
+  - service: weaver
+    resource: /providers/malleefowl
+    permission: read
+    group: anonymous
+    action: create
+  # raven
+  - service: weaver
+    resource: /providers/raven
+    permission: read
+    group: anonymous
+    action: create
+
+  # FIXME: remove when https://github.com/Ouranosinc/Magpie/issues/360 implemented
+  # for the moment, align WPS processes to allow public GetCapabilities and DescribeProcess like REST endpoint does
+  - service: weaver-wps
+    permission: getcapabilities
+    group: anonymous
+    action: create
+  - service: weaver-wps
+    permission: describeprocess
+    group: anonymous
+    action: create