Merge pull request #107 from bird-house/magpie-thredds

Magpie protected THREDDS
- add custom config to allow display of UI catalog elements of THREDDS service protected behind Magpie/Twitcher
- add test notebook that validates protected access of THREDDS files and directories behind Magpie/Twitcher auth
- bump Magpie/Twitcher versions aligned with Magpie 3.x to get custom config stored in db and THREDDS service updates

Successful build: http://daccs-jenkins.crim.ca/job/PAVICS-e2e-workflow-tests/job/magpie-func-tests/39/

birdhouse/config/magpie/providers.cfg.template

@@ -62,6 +62,32 @@ providers:
     c4i: false
     type: thredds
     sync_type: thredds
+    # below is a custom config to indicate how magpie should convert thredds path elements into resources/permissions
+    # see: https://pavics-magpie.readthedocs.io/en/latest/services.html#servicethredds
+    configuration:
+      skip_prefix: "thredds"  # prefix to ignore, below prefixes will be matched against whatever comes after in path
+      file_patterns:
+        # note: make sure to employ quotes and double escapes to avoid parsing YAML error
+        - ".+\\.ncml"  # match longest extension first to avoid tuncating it by match of sorter '.nc'
+        - ".+\\.nc"
+      metadata_type:
+        prefixes:
+          - null  # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed
+          - "\\w+\\.gif"  # threddsIcon, folder icon, etc.
+          - "\\w+\\.ico"  # favicon
+          - "\\w+\\.txt"  # licence
+          - "\\w+\\.css"  # tds.css
+          - "catalog\\.\\w+"  # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE`
+          - catalog
+          - ncml
+          - uddc
+          - iso
+      data_type:
+        prefixes:
+          - fileServer
+          - dodsC
+          - wcs
+          - wms
 
   ncWMS2:
     url: http://${PAVICS_FQDN}:8080/ncWMS2

birdhouse/docker-compose.yml

@@ -286,7 +286,7 @@ services:
     restart: always
 
   magpie:
-    image: pavics/magpie:1.7.3
+    image: pavics/magpie:3.5.1
     container_name: magpie
     ports:
       - "2001:2001"
@@ -312,7 +312,7 @@ services:
     restart: always
 
   twitcher:
-    image: pavics/twitcher:magpie-1.7.3
+    image: pavics/twitcher:magpie-3.5.1
     container_name: twitcher
     ports:
       - "8000:8000"

birdhouse/env.local.example

@@ -69,14 +69,17 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 # Format: space separated list of dirs
 #
 #export EXTRA_CONF_DIRS="/path/to/dir1 ./path/to/dir2 dir3 dir4"
-#export EXTRA_CONF_DIRS="./optional-components/canarie-api-full-monitoring
+#export EXTRA_CONF_DIRS="
+#    ./optional-components/canarie-api-full-monitoring
 #    ./optional-components/emu
 #    ./optional-components/testthredds
 #    ./optional-components/generic_bird
 #    ./optional-components/all-public-access
+#    ./optional-components/secure-thredds
 #    ./components/scheduler
 #    ./components/monitoring
-#    /path/to/private-config-repo"
+#    /path/to/private-config-repo
+#"
 
 # Extra repos, than the current repo, the autodeploy should keep up-to-date.
 # Any changes to these extra repos will also trigger autodeploy.

birdhouse/optional-components/README.rst

@@ -15,7 +15,8 @@ the end user).
 This assume all the WPS services are public.  If not the case, make a copy of
 this config and adjust accordingly.
 
-How to enable this config in ``env.local`` (a copy from env.local.example_ (:download:`download </birdhouse/env.local.example>`)):
+How to enable this config in ``env.local`` (a copy from env.local.example_
+(:download:`download </birdhouse/env.local.example>`)):
 
 * Add ``./optional-components/canarie-api-full-monitoring`` to ``EXTRA_CONF_DIRS``.
 
@@ -30,13 +31,15 @@ alternative configuration of existing birds.
 No Postgres DB configured.  If need Postgres DB, use generic_bird component
 instead.
 
-How to enable Emu in ``env.local`` (a copy from env.local.example_ (:download:`download </birdhouse/env.local.example>`)):
+How to enable Emu in ``env.local`` (a copy from env.local.example_
+(:download:`download </birdhouse/env.local.example>`)):
 
 * Add ``./optional-components/emu`` to ``EXTRA_CONF_DIRS``.
 * Optionally set ``EMU_IMAGE``, ``EMU_PORT``,
   ``EMU_NAME``, ``EMU_INTERNAL_PORT``,
   ``EMU_WPS_OUTPUTS_VOL`` in ``env.local`` for further customizations.
-  Default values are in `optional-components/emu/default.env <emu/default.env>`_ (:download:`download </birdhouse/optional-components/emu/default.env>`).
+  Default values are in `optional-components/emu/default.env <emu/default.env>`_
+  (:download:`download </birdhouse/optional-components/emu/default.env>`).
 
 Emu service will be available at ``http://PAVICS_FQDN:EMU_PORT/wps`` or
 ``https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/EMU_NAME`` where
@@ -50,7 +53,7 @@ CANARIE monitoring will also be automatically configured for this Emu WPS
 service.
 
 
-A second THREDDs server for testing
+A second THREDDS server for testing
 -----------------------------------
 
 How to enable in ``env.local`` (a copy from env.local.example_ (:download:`download </birdhouse/env.local.example>`)):
@@ -62,12 +65,12 @@ How to enable in ``env.local`` (a copy from env.local.example_ (:download:`downl
   ``TESTTHREDDS_INTERNAL_PORT``\ , ``TESTTHREDDS_NAME``\ ,  in ``env.local`` for further
   customizations.  Default values are in: `optional-components/testthredds/default.env <testthredds/default.env>`_ (:download:`download </birdhouse/optional-components/testthredds/default.env>`).
 
-Test THREDDs service will be available at
+Test THREDDS service will be available at
 ``http://PAVICS_FQDN:TESTTHREDDS_PORT/TESTTHREDDS_CONTEXT_ROOT`` or
 ``https://PAVICS_FQDN_PUBLIC/TESTTHREDDS_CONTEXT_ROOT`` where ``PAVICS_FQDN`` and
 ``PAVICS_FQDN_PUBLIC`` are defined in your ``env.local``.
 
-Use same docker image as regular THREDDs by default but can be customized.
+Use same docker image as regular THREDDS by default but can be customized.
 
 New container have new ``TestDatasets`` with volume-mount to ``/data/testdatasets``
 on the host.  So your testing ``.nc`` and ``.ncml`` files should be added to
@@ -79,11 +82,11 @@ server.
 ``export EMU_WPS_OUTPUTS_VOL=testwps_outputs`` to ``env.local`` for Emu to write to
 ``TestWps_Output`` dataset.
 
-No Twitcher/Magpie access control, this Test THREDDs is directly behind the
+No Twitcher/Magpie access control, this Test THREDDS is directly behind the
 Nginx proxy.
 
 CANARIE monitoring will also be automatically configured for this second
-THREDDs server.
+THREDDS server.
 
 
 A generic bird WPS service
@@ -99,7 +102,8 @@ How to enable in ``env.local`` (a copy from env.local.example_ (:download:`downl
 * Optionally set ``GENERIC_BIRD_IMAGE``, ``GENERIC_BIRD_PORT``,
   ``GENERIC_BIRD_NAME``, ``GENERIC_BIRD_INTERNAL_PORT``, and
   ``GENERIC_BIRD_POSTGRES_IMAGE`` in ``env.local`` for further customizations.
-  Default values are in `optional-components/generic_bird/default.env <generic_bird/default.env>`_ (:download:`download </birdhouse/optional-components/generic_bird/default.env>`).
+  Default values are in `optional-components/generic_bird/default.env <generic_bird/default.env>`_
+  (:download:`download </birdhouse/optional-components/generic_bird/default.env>`).
 
 The WPS service will be available at ``http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps``
 or ``https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME`` where
@@ -117,23 +121,52 @@ access for this WPS service.
 
 CANARIE monitoring will also be automatically configured for this WPS service.
 
+.. _magpie-public-access-config:
 
 Give public access to all resources for testing purposes
 --------------------------------------------------------
 
-By enabling this component, all WPS services and data on THREDDs are completely public, please beware.
-Once enabled, if you need to revert the change, you have to do it manually by logging into Magpie. 
+By enabling this component, all WPS services and data on THREDDS are completely public, please beware.
+Once enabled, if you need to revert the change, you have to do it manually by logging into Magpie.
 Just disabling this component will not revert the change.
+Alternatively, you can create a similar file to |magpie-public-perms|_ and replace all desired ``action: create``
+entries by ``action: remove`` to make sure the permissions are removed as startup if they exist.
 
 This optional component is required for the test suite at
 https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests.
 
-How to enable in ``env.local`` (a copy from env.local.example_ (:download:`download </birdhouse/env.local.example>`)):
+How to enable in ``env.local`` (a copy from `env.local.example`_ (:download:`download </birdhouse/env.local.example>`)):
 
 * Add ``./optional-components/all-public-access`` to ``EXTRA_CONF_DIRS``.
 
-The anonymous user will now have all the permissions described in `optional-components/all-public-access/all-public-access-magpie-permission.cfg <all-public-access/all-public-access-magpie-permission.cfg>`_ (:download:`download </birdhouse/optional-components/all-public-access/all-public-access-magpie-permission.cfg>`).
-
+The anonymous user will now have all the permissions described in |magpie-public-perms|_
+(:download:`download </birdhouse/optional-components/all-public-access/all-public-access-magpie-permission.cfg>`).
 
 
+.. _magpie-public-perms: ./all-public-access/all-public-access-magpie-permission.cfg
+.. |magpie-public-perms| replace:: optional-components/all-public-access/all-public-access-magpie-permission.cfg
 .. _env.local.example: ../env.local.example
+
+
+Control secured access to resources example
+--------------------------------------------------------
+
+Optional configuration |magpie-secure-perms|_ is provided as example to illustrate how to apply permissions on specific
+THREDDS resources to limit their access publicly. This permission configuration can be combined with others, such as
+`magpie-public-access-config`_ ones to formulate specific permissions schemes that matches your data structure and
+desired access rules.
+
+How to enable in ``env.local`` (a copy from `env.local.example`_ (:download:`download </birdhouse/env.local.example>`)):
+
+* Add ``./optional-components/secure-thredds`` to ``EXTRA_CONF_DIRS``.
+
+The anonymous user will *NOT* have access anymore to THREDDS test directory ``birdhouse/testdata/secure`` and any other
+directories and files under it. Directories above and next to ``secure`` will still be accessible if
+`magpie-public-access-config`_ component was also enabled.
+
+On a typical server, custom and private permission rules should be provided in a similar fashion to ensure that
+each time a new instance is booted, the same scheme of access configuration is applied. Permissions applied manually
+into Magpie will not be replicated onto other server instance.
+
+.. _magpie-secure-perms: ./secure-thredds/secure-access-magpie-permission.cfg
+.. |magpie-secure-perms| replace:: optional-components/secure-thredds/secure-access-magpie-permission.cfg

birdhouse/optional-components/all-public-access/all-public-access-magpie-permission.cfg

@@ -99,3 +99,8 @@ permissions:
     permission: read
     group: anonymous
     action: create
+
+  - service: thredds
+    permission: browse
+    group: anonymous
+    action: create

birdhouse/optional-components/secure-thredds/docker-compose-extra.yml

@@ -0,0 +1,5 @@
+version: '2.1'
+services:
+  magpie:
+    volumes:
+    - ./optional-components/secure-thredds/secure-access-magpie-permission.cfg:/opt/local/src/magpie/config/permissions/secure-access-magpie-permission.cfg:ro

birdhouse/optional-components/secure-thredds/secure-access-magpie-permission.cfg

@@ -0,0 +1,68 @@
+permissions:
+  # note:
+  #   following permissions can be combined with others such as 'optional-components/all-public-access'
+  #   to provide access to 'everything' except those under 'secure' directories listed below
+
+  # following permissions only enforce security on specific directories and files under it
+  # these can be reverted or combined with other set of permissions on resources 'above' or 'under' in the hierarchy
+  # users or groups will need explicit permissions under following resources for them to access sub-directories/files
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: browse
+      access: deny
+      scope: recursive
+    group: anonymous
+    action: create
+
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: read
+      access: deny
+      scope: recursive
+    group: anonymous
+    action: create
+
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: write
+      access: deny
+      scope: recursive
+    group: anonymous
+    action: create
+
+  # preserve access for test-suite user
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: browse
+      access: allow
+      scope: recursive
+    user: authtest
+    action: create
+
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: read
+      access: allow
+      scope: recursive
+    user: authtest
+    action: create
+
+  - service: thredds
+    resource: /birdhouse/testdata/secure
+    type: directory
+    permission:
+      name: write
+      access: allow
+      scope: recursive
+    user: authtest
+    action: create

birdhouse/scripts/create-magpie-users

@@ -98,7 +98,7 @@ if [ -z "$MAGPIE_CLI_CONF" ]; then
 fi
 
 if [ -z "$MAGPIE_CLI_IMAGE" ]; then
-    MAGPIE_CLI_IMAGE="pavics/magpie:2.0.1"
+    MAGPIE_CLI_IMAGE="pavics/magpie:3.2.0"
 fi
 
 # End configurable config via env var.