Adding SCP GUI

qiita_db/support_files/patches/python_patches/66.py

@@ -6,22 +6,6 @@
 from qiita_db.study import Study
 from re import sub
 
-# August 31, 2018
-# Strip any UTF-8 characters that are not also printable ASCII characters
-# from study titles. As some analysis packages cannot interpret UTF-8
-# characters, it becomes important to remove them from study titles, as
-# they are used as metadata/identifiers when creating new analyses.
-
-# insert new status_types into list, or replace w/a call to an appropriate
-# method.
-status_types = ['awaiting_approval', 'sandbox', 'private', 'public']
-
-for status_type in status_types:
-    for study in Study.get_by_status(status_type):
-        new_title = sub(r'[^\x20-\x7E]+', '', study.title)
-        if new_title != study.title:
-            study.title = new_title
-
 
 # August 6, 2018
 # Create parameters for the ssh/scp remote file upload commands
@@ -168,7 +152,8 @@ def create_command(software, name, description, parameters, outputs=None,
 
     # Create the 'list_remote_files' command
     parameters = {'url': ['string', None],
-                  'private_key': ['string', None]}
+                  'private_key': ['string', None],
+                  'study_id': ['integer', None]}
     create_command(qiita_plugin, "list_remote_files",
                    "retrieves list of valid study files from remote dir",
                    parameters)
@@ -179,3 +164,20 @@ def create_command(software, name, description, parameters, outputs=None,
                   'private_key': ['string', None]}
     create_command(qiita_plugin, "download_remote_files",
                    "downloads valid study files from remote dir", parameters)
+
+
+# August 31, 2018
+# Strip any UTF-8 characters that are not also printable ASCII characters
+# from study titles. As some analysis packages cannot interpret UTF-8
+# characters, it becomes important to remove them from study titles, as
+# they are used as metadata/identifiers when creating new analyses.
+
+# insert new status_types into list, or replace w/a call to an appropriate
+# method.
+status_types = ['awaiting_approval', 'sandbox', 'private', 'public']
+
+for status_type in status_types:
+    for study in Study.get_by_status(status_type):
+        new_title = sub(r'[^\x20-\x7E]+', '', study.title)
+        if new_title != study.title:
+            study.title = new_title

qiita_pet/handlers/upload.py

@@ -10,20 +10,27 @@
 
 from os.path import isdir, join, exists
 from os import makedirs, remove
+from json import loads, dumps
 
+from collections import defaultdict
 from shutil import rmtree, move
 
 from .util import check_access
 from .base_handlers import BaseHandler
 
-from qiita_core.qiita_settings import qiita_config
+from qiita_core.qiita_settings import qiita_config, r_client
 from qiita_core.util import execute_as_transaction
 from qiita_db.util import (get_files_from_uploads_folders,
                            get_mountpoint, move_upload_files_to_trash)
 from qiita_db.study import Study
+from qiita_db.processing_job import ProcessingJob
+from qiita_db.software import Software, Parameters
 from qiita_db.exceptions import QiitaDBUnknownIDError
 
 
+UPLOAD_STUDY_FORMAT = 'upload_study_%s'
+
+
 class StudyUploadFileHandler(BaseHandler):
     @authenticated
     @execute_as_transaction
@@ -32,14 +39,45 @@ def display_template(self, study_id, msg):
         study_id = int(study_id)
         study = Study(study_id)
         user = self.current_user
+        level = 'info'
+        message = ''
+        remote_url = ''
+        remote_files = []
         check_access(user, study, no_public=True, raise_error=True)
 
+        job_info = r_client.get(UPLOAD_STUDY_FORMAT % study_id)
+        if job_info:
+            job_info = defaultdict(lambda: '', loads(job_info))
+            job_id = job_info['job_id']
+            job = ProcessingJob(job_id)
+            job_status = job.status
+            processing = job_status not in ('success', 'error')
+            url = job.parameters.values['url']
+            if processing:
+                if job.command.name == 'list_remote_files':
+                    message = 'Retrieving remote files: listing %s' % url
+                else:
+                    message = 'Retrieving remote files: download %s' % url
+            elif job_status == 'error':
+                level = 'danger'
+                message = job.log.msg.replace('\n', '</br>')
+                # making errors nicer for users
+                if 'No such file' in message:
+                    message = 'URL not valid: <i>%s</i>, please review.' % url
+            else:
+                remote_url = job_info['url']
+                remote_files = job_info['files']
+                level = job_info['alert_type']
+                message = job_info['alert_msg'].replace('\n', '</br>')
+
         # getting the ontologies
         self.render('upload.html',
                     study_title=study.title, study_info=study.info,
                     study_id=study_id, is_admin=user.level == 'admin',
                     extensions=','.join(qiita_config.valid_upload_extension),
-                    max_upload_size=qiita_config.max_upload_size,
+                    max_upload_size=qiita_config.max_upload_size, level=level,
+                    message=message, remote_url=remote_url,
+                    remote_files=remote_files,
                     files=get_files_from_uploads_folders(str(study_id)))
 
     @authenticated
@@ -78,6 +116,53 @@ def post(self, study_id):
         self.display_template(study_id, "")
 
 
+class StudyUploadViaRemote(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def post(self, study_id):
+        method = self.get_argument('remote-request-type')
+        url = self.get_argument('inputURL')
+        ssh_key = self.request.files['ssh-key'][0]['body']
+        status = 'success'
+        message = ''
+
+        try:
+            study = Study(int(study_id))
+        except QiitaDBUnknownIDError:
+            raise HTTPError(404, reason="Study %s does not exist" % study_id)
+        check_access(
+            self.current_user, study, no_public=True, raise_error=True)
+
+        _, upload_folder = get_mountpoint("uploads")[0]
+        upload_folder = join(upload_folder, study_id)
+        ssh_key_fp = join(upload_folder, '.key.txt')
+
+        with open(ssh_key_fp, 'w') as f:
+            f.write(ssh_key)
+
+        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
+        if method == 'list':
+            cmd = qiita_plugin.get_command('list_remote_files')
+            params = Parameters.load(cmd, values_dict={
+                'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
+        elif method == 'transfer':
+            cmd = qiita_plugin.get_command('download_remote_files')
+            params = Parameters.load(cmd, values_dict={
+                'url': url, 'private_key': ssh_key_fp,
+                'destination': upload_folder})
+        else:
+            status = 'error'
+            message = 'Not a valid method'
+
+        if status == 'success':
+            job = ProcessingJob.create(self.current_user, params, True)
+            job.submit()
+            r_client.set(
+                UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))
+
+        self.write({'status': status, 'message': message})
+
+
 class UploadFileHandler(BaseHandler):
     # """ main upload class
     # based on