Separate plugin file system from BASA_DATA_DIR (#3480)

sjanssen2 · web-flow · commit 1b8b0c71a52a · 2025-08-27T09:50:40.000-06:00
* extended configuration manager with optional OIDC sections

* flake8

* also provide a label for a speaking name of the identity provider

* start implementing the OIDC dance

* modal not necessary, if only one provider was defined

* error handling of provider not in config file

* adding pycurl package to enable tornado curl_httpclients

* a new method to create a user, if information do not need to be entered manually, but are obtained from an external identity provider

* full OIDC dance implemented

* add an admin page to activate users which requested authorization through OIDC

* flake8

* adding menu entry for user authorization

* do not expose traditional qiita internal user authentication, if OIDC is configured

* use Qiita typical modal for OIDC login

* wrong menu entrie affected

* always allow logout

* improved error handling

* revert: let user change their profile, but not password - if provided through OIDC

* speaking button names + move into correct div to always get displayed

* use email from config + loop user_info from OIDC to fill DB

* use OIDC info to prefil user information

* drop admin user authorization

* using the well-known json dict instead of manually providing multiple API endpoints through the config file

* flake8

* flake8

* add ability to display OIDC logos

* add OIDC logo

* revert to dev branch

* fixing config manager tests

* add missing template

* extended configuration manager with optional OIDC sections

* flake8

* also provide a label for a speaking name of the identity provider

* start implementing the OIDC dance

* modal not necessary, if only one provider was defined

* error handling of provider not in config file

* adding pycurl package to enable tornado curl_httpclients

* a new method to create a user, if information do not need to be entered manually, but are obtained from an external identity provider

* full OIDC dance implemented

* add an admin page to activate users which requested authorization through OIDC

* flake8

* adding menu entry for user authorization

* do not expose traditional qiita internal user authentication, if OIDC is configured

* use Qiita typical modal for OIDC login

* always allow logout

* improved error handling

* revert: let user change their profile, but not password - if provided through OIDC

* speaking button names + move into correct div to always get displayed

* use email from config + loop user_info from OIDC to fill DB

* use OIDC info to prefil user information

* drop admin user authorization

* using the well-known json dict instead of manually providing multiple API endpoints through the config file

* flake8

* flake8

* add ability to display OIDC logos

* add OIDC logo

* fixing config manager tests

* multiple validation jobs should be submitted as lead and dependent jobs, but the later must also made known by the DB

* expose additional HTTP endpoints to send and retrieve data over http from/to plugin

* add another configuration variable to decide if HTTPS filetransfer endpoint will be exposed

* aim to subtract OIDC changes

* remove image

* restore blank line

* revert changes

* revert

* revert

* return info as "reason"

* initial set of tests

* codestyle

* more codestyle

* enable endpoints

* tests pass locally

* adding in more functionallity for testing, i.e. operate on token based authorization

* avoid name collisions

* debug

* more debug

* change debug

* continue debug

* adapt filepaths to github runner

* remove debug and fix codestyle

* remove debug step

* after talking with Antonio, we decided that no additional config parameter for ENABLE_HTTPS_PLUGIN_FILETRANSFER is necessary, we simply expose these new endpoints, whatever

* Update config_test.cfg
diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml
@@ -148,6 +148,7 @@ jobs:
 
           echo "4. Setting up nginx"
           mkdir -p /usr/share/miniconda/envs/qiita/var/run/nginx/
+          sed -i "s|alias /Users/username|alias /home/runner/work/qiita|" ${PWD}/qiita_pet/nginx_example.conf
           nginx -c ${PWD}/qiita_pet/nginx_example.conf
 
           echo "5. Setting up qiita"
diff --git a/qiita_db/handlers/tests/oauthbase.py b/qiita_db/handlers/tests/oauthbase.py
@@ -6,7 +6,10 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-from qiita_core.qiita_settings import r_client
+import requests
+import os
+import sys
+from qiita_core.qiita_settings import r_client, qiita_config
 
 from qiita_pet.test.tornado_test_base import TestHandlerBase
 
@@ -19,3 +22,49 @@ def setUp(self):
         r_client.hset(self.token, 'grant_type', 'client')
         r_client.expire(self.token, 20)
         super(OauthTestingBase, self).setUp()
+        self._session = requests.Session()
+        # should point to client certificat file:
+        # /qiita/qiita_core/support_files/ci_rootca.crt
+        self._verify = os.environ['QIITA_ROOTCA_CERT']
+        self._fetch_token()
+
+        self._files_to_remove = []
+
+    def tearDown(self):
+        for fp in self._files_to_remove:
+            if os.path.exists(fp):
+                os.remove(fp)
+
+    def _fetch_token(self):
+        data = {
+            'client_id': '4MOBzUBHBtUmwhaC258H7PS0rBBLyGQrVxGPgc9g305bvVhf6h',
+            'client_secret':
+            ('rFb7jwAb3UmSUN57Bjlsi4DTl2owLwRpwCc0SggRN'
+             'EVb2Ebae2p5Umnq20rNMhmqN'),
+            'grant_type': 'client'}
+        resp = self._session.post(
+            "%s/qiita_db/authenticate/" % qiita_config.base_url,
+            verify=self._verify, data=data, timeout=80)
+        if resp.status_code != 200:
+            raise ValueError("_fetchToken() POST request failed")
+        self._token = resp.json()['access_token']
+        print('obtained access_token = %s' % self._token, file=sys.stderr)
+
+    def post_authed(self, url, **kwargs):
+        if 'headers' not in kwargs:
+            kwargs['headers'] = {}
+        if 'Authorization' not in kwargs['headers']:
+            kwargs['headers']['Authorization'] = 'Bearer %s' % self._token
+
+        r = self._session.post(
+            qiita_config.base_url + url, verify=self._verify, **kwargs)
+        r.close()
+
+        return r
+
+    def get_authed(self, url):
+        r = self._session.get(qiita_config.base_url + url, verify=self._verify,
+                              headers={'Authorization': 'Bearer %s' %
+                                       self._token})
+        r.close()
+        return r
diff --git a/qiita_pet/handlers/cloud_handlers/__init__.py b/qiita_pet/handlers/cloud_handlers/__init__.py
@@ -0,0 +1,9 @@
+from .file_transfer_handlers import (FetchFileFromCentralHandler,
+                                     PushFileToCentralHandler)
+
+__all__ = ['FetchFileFromCentralHandler']
+
+ENDPOINTS = [
+    (r"/cloud/fetch_file_from_central/(.*)", FetchFileFromCentralHandler),
+    (r"/cloud/push_file_to_central/", PushFileToCentralHandler)
+]
diff --git a/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/file_transfer_handlers.py
@@ -0,0 +1,94 @@
+import os
+
+from tornado.web import HTTPError, RequestHandler
+from tornado.gen import coroutine
+
+from qiita_core.util import execute_as_transaction
+from qiita_db.handlers.oauth2 import authenticate_oauth
+from qiita_core.qiita_settings import qiita_config
+
+
+class FetchFileFromCentralHandler(RequestHandler):
+    @authenticate_oauth
+    @coroutine
+    @execute_as_transaction
+    def get(self, requested_filepath):
+        # ensure we have an absolute path, i.e. starting at /
+        filepath = os.path.join(os.path.sep, requested_filepath)
+        # use a canonic version of the filepath
+        filepath = os.path.abspath(filepath)
+
+        # canonic version of base_data_dir
+        basedatadir = os.path.abspath(qiita_config.base_data_dir)
+
+        # TODO: can we somehow check, if the requesting client (which should be
+        #       one of the plugins) was started from a user that actually has
+        #       access to the requested file?
+
+        if not filepath.startswith(basedatadir):
+            # attempt to access files outside of the BASE_DATA_DIR
+            # intentionally NOT reporting the actual location to avoid exposing
+            # instance internal information
+            raise HTTPError(403, reason=(
+                "You cannot access files outside of "
+                "the BASE_DATA_DIR of Qiita!"))
+
+        if not os.path.exists(filepath):
+            raise HTTPError(403, reason=(
+                "The requested file is not present in Qiita's BASE_DATA_DIR!"))
+
+        # delivery of the file via nginx requires replacing the basedatadir
+        # with the prefix defined in the nginx configuration for the
+        # base_data_dir, '/protected/' by default
+        protected_filepath = filepath.replace(basedatadir, '/protected')
+
+        self.set_header('Content-Type', 'application/octet-stream')
+        self.set_header('Content-Transfer-Encoding', 'binary')
+        self.set_header('X-Accel-Redirect', protected_filepath)
+        self.set_header('Content-Description', 'File Transfer')
+        self.set_header('Expires',  '0')
+        self.set_header('Cache-Control',  'no-cache')
+        self.set_header('Content-Disposition',
+                        'attachment; filename=%s' % os.path.basename(
+                            protected_filepath))
+        self.finish()
+
+
+class PushFileToCentralHandler(RequestHandler):
+    @authenticate_oauth
+    @coroutine
+    @execute_as_transaction
+    def post(self):
+        if not self.request.files:
+            raise HTTPError(400, reason='No files to upload defined!')
+
+        # canonic version of base_data_dir
+        basedatadir = os.path.abspath(qiita_config.base_data_dir)
+        stored_files = []
+
+        for filespath, filelist in self.request.files.items():
+            if filespath.startswith(basedatadir):
+                filespath = filespath[len(basedatadir):]
+
+            for file in filelist:
+                filepath = os.path.join(filespath, file['filename'])
+                # remove leading /
+                if filepath.startswith(os.sep):
+                    filepath = filepath[len(os.sep):]
+                filepath = os.path.abspath(os.path.join(basedatadir, filepath))
+
+                if os.path.exists(filepath):
+                    raise HTTPError(403, reason=(
+                        "The requested file is already "
+                        "present in Qiita's BASE_DATA_DIR!"))
+
+                os.makedirs(os.path.dirname(filepath), exist_ok=True)
+                with open(filepath, "wb") as f:
+                    f.write(file['body'])
+                    stored_files.append(filepath)
+
+        self.write("Stored %i files into BASE_DATA_DIR of Qiita:\n%s\n" % (
+            len(stored_files),
+            '\n'.join(map(lambda x: ' - %s' % x, stored_files))))
+
+        self.finish()
diff --git a/qiita_pet/handlers/cloud_handlers/tests/test_file_transfer_handlers.py b/qiita_pet/handlers/cloud_handlers/tests/test_file_transfer_handlers.py
@@ -0,0 +1,93 @@
+from unittest import main
+from os.path import exists, basename
+from os import remove
+import filecmp
+
+from qiita_db.handlers.tests.oauthbase import OauthTestingBase
+import qiita_db as qdb
+
+
+class FetchFileFromCentralHandlerTests(OauthTestingBase):
+    def setUp(self):
+        super(FetchFileFromCentralHandlerTests, self).setUp()
+
+    def test_get(self):
+        endpoint = '/cloud/fetch_file_from_central/'
+        base_data_dir = qdb.util.get_db_files_base_dir()
+
+        obs = self.get_authed(endpoint + 'nonexistingfile')
+        self.assertEqual(obs.status_code, 403)
+        self.assertIn('outside of the BASE_DATA_DIR', obs.reason)
+
+        obs = self.get_authed(
+            endpoint + base_data_dir[1:] + '/nonexistingfile')
+        self.assertEqual(obs.status_code, 403)
+        self.assertIn('The requested file is not present', obs.reason)
+
+        obs = self.get_authed(
+            endpoint + base_data_dir[1:] +
+            '/raw_data/FASTA_QUAL_preprocessing.fna')
+        self.assertEqual(obs.status_code, 200)
+        self.assertIn('FLP3FBN01ELBSX length=250 xy=1766_01', str(obs.content))
+
+
+class PushFileToCentralHandlerTests(OauthTestingBase):
+    def setUp(self):
+        super(PushFileToCentralHandlerTests, self).setUp()
+
+    def test_post(self):
+        endpoint = '/cloud/push_file_to_central/'
+        base_data_dir = qdb.util.get_db_files_base_dir()
+
+        # create a test file "locally", i.e. in current working directory
+        fp_source = 'foo.bar'
+        with open(fp_source, 'w') as f:
+            f.write("this is a test\n")
+        self._files_to_remove.append(fp_source)
+
+        # if successful, expected location of the file in BASE_DATA_DIR
+        fp_target = base_data_dir + '/bar/' + basename(fp_source)
+        self._files_to_remove.append(fp_target)
+
+        # create a second test file
+        fp_source2 = 'foo_two.bar'
+        with open(fp_source2, 'w') as f:
+            f.write("this is another test\n")
+        self._files_to_remove.append(fp_source2)
+        fp_target2 = base_data_dir + '/barr/' + basename(fp_source2)
+        self._files_to_remove.append(fp_target2)
+
+        # test raise error if no file is given
+        obs = self.post_authed(endpoint)
+        self.assertEqual(obs.reason, "No files to upload defined!")
+
+        # test correct mechanism
+        with open(fp_source, 'rb') as fh:
+            obs = self.post_authed(endpoint, files={'bar/': fh})
+            self.assertIn('Stored 1 files into BASE_DATA_DIR of Qiita',
+                          str(obs.content))
+            self.assertTrue(filecmp.cmp(fp_source, fp_target, shallow=False))
+
+        # check if error is raised, if file already exists
+        with open(fp_source, 'rb') as fh:
+            obs = self.post_authed(endpoint, files={'bar/': fh})
+            self.assertIn("already present in Qiita's BASE_DATA_DIR!",
+                          obs.reason)
+
+        # test transfer of multiple files
+        if exists(fp_target):
+            remove(fp_target)
+        with open(fp_source, 'rb') as fh1:
+            with open(fp_source2, 'rb') as fh2:
+                obs = self.post_authed(
+                    endpoint, files={'bar/': fh1, 'barr/': fh2})
+                self.assertIn('Stored 2 files into BASE_DATA_DIR of Qiita',
+                              str(obs.content))
+                self.assertTrue(filecmp.cmp(fp_source, fp_target,
+                                            shallow=False))
+                self.assertTrue(filecmp.cmp(fp_source2, fp_target2,
+                                            shallow=False))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py
@@ -85,6 +85,7 @@
 from qiita_pet.handlers.rest import ENDPOINTS as REST_ENDPOINTS
 from qiita_pet.handlers.qiita_redbiom import RedbiomPublicSearch
 from qiita_pet.handlers.public import PublicHandler
+from qiita_pet.handlers.cloud_handlers import ENDPOINTS as CLOUD_ENDPOINTS
 
 if qiita_config.portal == "QIITA":
     from qiita_pet.handlers.portal import (
@@ -244,6 +245,11 @@ def __init__(self):
             (r"/qiita_db/studies/(.*)", APIStudiesListing)
         ]
 
+        # expose endpoints necessary for https file communication between
+        # master and plugins IF no shared file system for base_data_dir is
+        # intended
+        handlers.extend(CLOUD_ENDPOINTS)
+
         # rest endpoints
         handlers.extend(REST_ENDPOINTS)
 
