Separate plugin file system from BASA_DATA_DIR

.github/workflows/qiita-ci.yml

@@ -58,6 +58,9 @@ jobs:
           pgport=${{ job.services.postgres.ports[5432] }}
           sed -i "s/PORT = 5432/PORT = $pgport/" qiita_core/support_files/config_test.cfg
 
+          # enable endpoints for HTTPS file transfer from/to plugin(s) and qiita main
+          sed -i "s/ENABLE_HTTPS_PLUGIN_FILETRANSFER = False/ENABLE_HTTPS_PLUGIN_FILETRANSFER = True/" qiita_core/support_files/config_test.cfg
+
           # PGPASSWORD is read by pg_restore, which is called by the build_db process.
           export PGPASSWORD=postgres
 
@@ -148,6 +151,7 @@ jobs:
 
           echo "4. Setting up nginx"
           mkdir -p /usr/share/miniconda/envs/qiita/var/run/nginx/
+          sed -i "s|alias /Users/username|alias /home/runner/work/qiita|" ${PWD}/qiita_pet/nginx_example.conf
           nginx -c ${PWD}/qiita_pet/nginx_example.conf
 
           echo "5. Setting up qiita"

qiita_core/configuration_manager.py

@@ -123,6 +123,12 @@ class ConfigurationManager(object):
         The script used to start the plugins
     plugin_dir : str
         The path to the directory containing the plugin configuration files
+    enable_https_plugin_filetransfer : bool
+        Allow BASE_DATA_DIR file content transfer through https (True or False)
+        By default, Qiita and its plugins share the filesystem of
+        BASE_DATA_DIR. You can less tightly couple selected plugins
+        (=no shared file system) but they need to get/push input/output files
+        through https then
     help_email : str
         The email address a user should write to when asking for help
     sysadmin_email : str
@@ -215,6 +221,8 @@ def _get_main(self, config):
         elif not isdir(self.plugin_dir):
             raise ValueError("The PLUGIN_DIR (%s) folder doesn't exist"
                              % self.plugin_dir)
+        self.enable_https_plugin_filetransfer = config.getboolean(
+            'main', 'ENABLE_HTTPS_PLUGIN_FILETRANSFER', fallback=False)
 
         self.valid_upload_extension = [ve.strip() for ve in config.get(
             'main', 'VALID_UPLOAD_EXTENSION').split(',')]

qiita_core/support_files/config_test.cfg

@@ -50,6 +50,12 @@ PLUGIN_LAUNCHER = qiita-plugin-launcher
 # Plugins configuration directory
 PLUGIN_DIR =
 
+# Allow BASE_DATA_DIR file content transfer through https (True or False)
+# By default, Qiita and its plugins share the filesystem of BASE_DATA_DIR.
+# You can less tightly couple selected plugins (=no shared file system) but
+# they need to get/push input/output files through https then
+ENABLE_HTTPS_PLUGIN_FILETRANSFER = False
+
 # Webserver certificate file paths
 CERTIFICATE_FILE =
 KEY_FILE =

qiita_core/tests/test_configuration_manager.py

@@ -55,6 +55,7 @@ def test_init(self):
         self.assertEqual(
             obs.valid_upload_extension,
             ["fastq", "fastq.gz", "txt", "tsv", "sff", "fna", "qual"])
+        self.assertEqual(obs.enable_https_plugin_filetransfer, False)
         self.assertEqual(obs.certificate_file, "/tmp/server.cert")
         self.assertEqual(obs.cookie_secret, "SECRET")
         self.assertEqual(obs.key_file, "/tmp/server.key")
@@ -331,6 +332,12 @@ def test_get_portal_latlong(self):
 # Plugins configuration directory
 PLUGIN_DIR = /tmp/
 
+# Allow BASE_DATA_DIR file content transfer through https (True or False)
+# By default, Qiita and its plugins share the filesystem of BASE_DATA_DIR.
+# You can less tightly couple selected plugins (=no shared file system) but
+# they need to get/push input/output files through https then
+ENABLE_HTTPS_PLUGIN_FILETRANSFER = False
+
 # Webserver certificate file paths
 CERTIFICATE_FILE = /tmp/server.cert
 KEY_FILE = /tmp/server.key

qiita_db/handlers/tests/oauthbase.py

@@ -6,7 +6,10 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-from qiita_core.qiita_settings import r_client
+import requests
+import os
+import sys
+from qiita_core.qiita_settings import r_client, qiita_config
 
 from qiita_pet.test.tornado_test_base import TestHandlerBase
 
@@ -19,3 +22,49 @@ def setUp(self):
         r_client.hset(self.token, 'grant_type', 'client')
         r_client.expire(self.token, 20)
         super(OauthTestingBase, self).setUp()
+        self._session = requests.Session()
+        # should point to client certificat file:
+        # /qiita/qiita_core/support_files/ci_rootca.crt
+        self._verify = os.environ['QIITA_ROOTCA_CERT']
+        self._fetch_token()
+
+        self._files_to_remove = []
+
+    def tearDown(self):
+        for fp in self._files_to_remove:
+            if os.path.exists(fp):
+                os.remove(fp)
+
+    def _fetch_token(self):
+        data = {
+            'client_id': '4MOBzUBHBtUmwhaC258H7PS0rBBLyGQrVxGPgc9g305bvVhf6h',
+            'client_secret':
+            ('rFb7jwAb3UmSUN57Bjlsi4DTl2owLwRpwCc0SggRN'
+             'EVb2Ebae2p5Umnq20rNMhmqN'),
+            'grant_type': 'client'}
+        resp = self._session.post(
+            "%s/qiita_db/authenticate/" % qiita_config.base_url,
+            verify=self._verify, data=data, timeout=80)
+        if resp.status_code != 200:
+            raise ValueError("_fetchToken() POST request failed")
+        self._token = resp.json()['access_token']
+        print('obtained access_token = %s' % self._token, file=sys.stderr)
+
+    def post_authed(self, url, **kwargs):
+        if 'headers' not in kwargs:
+            kwargs['headers'] = {}
+        if 'Authorization' not in kwargs['headers']:
+            kwargs['headers']['Authorization'] = 'Bearer %s' % self._token
+
+        r = self._session.post(
+            qiita_config.base_url + url, verify=self._verify, **kwargs)
+        r.close()
+
+        return r
+
+    def get_authed(self, url):
+        r = self._session.get(qiita_config.base_url + url, verify=self._verify,
+                              headers={'Authorization': 'Bearer %s' %
+                                       self._token})
+        r.close()
+        return r

qiita_pet/handlers/cloud_handlers/__init__.py

@@ -0,0 +1,9 @@
+from .file_transfer_handlers import (FetchFileFromCentralHandler,
+                                     PushFileToCentralHandler)
+
+__all__ = ['FetchFileFromCentralHandler']
+
+ENDPOINTS = [
+    (r"/cloud/fetch_file_from_central/(.*)", FetchFileFromCentralHandler),
+    (r"/cloud/push_file_to_central/", PushFileToCentralHandler)
+]

qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py

@@ -0,0 +1,94 @@
+import os
+
+from tornado.web import HTTPError, RequestHandler
+from tornado.gen import coroutine
+
+from qiita_core.util import execute_as_transaction
+from qiita_db.handlers.oauth2 import authenticate_oauth
+from qiita_core.qiita_settings import qiita_config
+
+
+class FetchFileFromCentralHandler(RequestHandler):
+    @authenticate_oauth
+    @coroutine
+    @execute_as_transaction
+    def get(self, requested_filepath):
+        # ensure we have an absolute path, i.e. starting at /
+        filepath = os.path.join(os.path.sep, requested_filepath)
+        # use a canonic version of the filepath
+        filepath = os.path.abspath(filepath)
+
+        # canonic version of base_data_dir
+        basedatadir = os.path.abspath(qiita_config.base_data_dir)
+
+        # TODO: can we somehow check, if the requesting client (which should be
+        #       one of the plugins) was started from a user that actually has
+        #       access to the requested file?
+
+        if not filepath.startswith(basedatadir):
+            # attempt to access files outside of the BASE_DATA_DIR
+            # intentionally NOT reporting the actual location to avoid exposing
+            # instance internal information
+            raise HTTPError(403, reason=(
+                "You cannot access files outside of "
+                "the BASE_DATA_DIR of Qiita!"))
+
+        if not os.path.exists(filepath):
+            raise HTTPError(403, reason=(
+                "The requested file is not present in Qiita's BASE_DATA_DIR!"))
+
+        # delivery of the file via nginx requires replacing the basedatadir
+        # with the prefix defined in the nginx configuration for the
+        # base_data_dir, '/protected/' by default
+        protected_filepath = filepath.replace(basedatadir, '/protected')
+
+        self.set_header('Content-Type', 'application/octet-stream')
+        self.set_header('Content-Transfer-Encoding', 'binary')
+        self.set_header('X-Accel-Redirect', protected_filepath)
+        self.set_header('Content-Description', 'File Transfer')
+        self.set_header('Expires',  '0')
+        self.set_header('Cache-Control',  'no-cache')
+        self.set_header('Content-Disposition',
+                        'attachment; filename=%s' % os.path.basename(
+                            protected_filepath))
+        self.finish()
+
+
+class PushFileToCentralHandler(RequestHandler):
+    @authenticate_oauth
+    @coroutine
+    @execute_as_transaction
+    def post(self):
+        if not self.request.files:
+            raise HTTPError(400, reason='No files to upload defined!')
+
+        # canonic version of base_data_dir
+        basedatadir = os.path.abspath(qiita_config.base_data_dir)
+        stored_files = []
+
+        for filespath, filelist in self.request.files.items():
+            if filespath.startswith(basedatadir):
+                filespath = filespath[len(basedatadir):]
+
+            for file in filelist:
+                filepath = os.path.join(filespath, file['filename'])
+                # remove leading /
+                if filepath.startswith(os.sep):
+                    filepath = filepath[len(os.sep):]
+                filepath = os.path.abspath(os.path.join(basedatadir, filepath))
+
+                if os.path.exists(filepath):
+                    raise HTTPError(403, reason=(
+                        "The requested file is already "
+                        "present in Qiita's BASE_DATA_DIR!"))
+
+                os.makedirs(os.path.dirname(filepath), exist_ok=True)
+                with open(filepath, "wb") as f:
+                    f.write(file['body'])
+                    stored_files.append(filepath)
+
+        self.write("Stored %i files into BASE_DATA_DIR of Qiita:\n%s\n" % (
+            len(stored_files),
+            '\n'.join(map(lambda x: ' - %s' % x, stored_files))))
+
+        self.finish()

qiita_pet/handlers/cloud_handlers/tests/test_file_transfer_handlers.py

@@ -0,0 +1,93 @@
+from unittest import main
+from os.path import exists, basename
+from os import remove
+import filecmp
+
+from qiita_db.handlers.tests.oauthbase import OauthTestingBase
+import qiita_db as qdb
+
+
+class FetchFileFromCentralHandlerTests(OauthTestingBase):
+    def setUp(self):
+        super(FetchFileFromCentralHandlerTests, self).setUp()
+
+    def test_get(self):
+        endpoint = '/cloud/fetch_file_from_central/'
+        base_data_dir = qdb.util.get_db_files_base_dir()
+
+        obs = self.get_authed(endpoint + 'nonexistingfile')
+        self.assertEqual(obs.status_code, 403)
+        self.assertIn('outside of the BASE_DATA_DIR', obs.reason)
+
+        obs = self.get_authed(
+            endpoint + base_data_dir[1:] + '/nonexistingfile')
+        self.assertEqual(obs.status_code, 403)
+        self.assertIn('The requested file is not present', obs.reason)
+
+        obs = self.get_authed(
+            endpoint + base_data_dir[1:] +
+            '/raw_data/FASTA_QUAL_preprocessing.fna')
+        self.assertEqual(obs.status_code, 200)
+        self.assertIn('FLP3FBN01ELBSX length=250 xy=1766_01', str(obs.content))
+
+
+class PushFileToCentralHandlerTests(OauthTestingBase):
+    def setUp(self):
+        super(PushFileToCentralHandlerTests, self).setUp()
+
+    def test_post(self):
+        endpoint = '/cloud/push_file_to_central/'
+        base_data_dir = qdb.util.get_db_files_base_dir()
+
+        # create a test file "locally", i.e. in current working directory
+        fp_source = 'foo.bar'
+        with open(fp_source, 'w') as f:
+            f.write("this is a test\n")
+        self._files_to_remove.append(fp_source)
+
+        # if successful, expected location of the file in BASE_DATA_DIR
+        fp_target = base_data_dir + '/bar/' + basename(fp_source)
+        self._files_to_remove.append(fp_target)
+
+        # create a second test file
+        fp_source2 = 'foo_two.bar'
+        with open(fp_source2, 'w') as f:
+            f.write("this is another test\n")
+        self._files_to_remove.append(fp_source2)
+        fp_target2 = base_data_dir + '/barr/' + basename(fp_source2)
+        self._files_to_remove.append(fp_target2)
+
+        # test raise error if no file is given
+        obs = self.post_authed(endpoint)
+        self.assertEqual(obs.reason, "No files to upload defined!")
+
+        # test correct mechanism
+        with open(fp_source, 'rb') as fh:
+            obs = self.post_authed(endpoint, files={'bar/': fh})
+            self.assertIn('Stored 1 files into BASE_DATA_DIR of Qiita',
+                          str(obs.content))
+            self.assertTrue(filecmp.cmp(fp_source, fp_target, shallow=False))
+
+        # check if error is raised, if file already exists
+        with open(fp_source, 'rb') as fh:
+            obs = self.post_authed(endpoint, files={'bar/': fh})
+            self.assertIn("already present in Qiita's BASE_DATA_DIR!",
+                          obs.reason)
+
+        # test transfer of multiple files
+        if exists(fp_target):
+            remove(fp_target)
+        with open(fp_source, 'rb') as fh1:
+            with open(fp_source2, 'rb') as fh2:
+                obs = self.post_authed(
+                    endpoint, files={'bar/': fh1, 'barr/': fh2})
+                self.assertIn('Stored 2 files into BASE_DATA_DIR of Qiita',
+                              str(obs.content))
+                self.assertTrue(filecmp.cmp(fp_source, fp_target,
+                                            shallow=False))
+                self.assertTrue(filecmp.cmp(fp_source2, fp_target2,
+                                            shallow=False))
+
+
+if __name__ == "__main__":
+    main()

qiita_pet/webserver.py

@@ -85,6 +85,7 @@
 from qiita_pet.handlers.rest import ENDPOINTS as REST_ENDPOINTS
 from qiita_pet.handlers.qiita_redbiom import RedbiomPublicSearch
 from qiita_pet.handlers.public import PublicHandler
+from qiita_pet.handlers.cloud_handlers import ENDPOINTS as CLOUD_ENDPOINTS
 
 if qiita_config.portal == "QIITA":
     from qiita_pet.handlers.portal import (
@@ -244,6 +245,12 @@ def __init__(self):
             (r"/qiita_db/studies/(.*)", APIStudiesListing)
         ]
 
+        # expose endpoints necessary for https file communication between
+        # master and plugins IF no shared file system for base_data_dir is
+        # intended
+        if qiita_config.enable_https_plugin_filetransfer:
+            handlers.extend(CLOUD_ENDPOINTS)
+
         # rest endpoints
         handlers.extend(REST_ENDPOINTS)