Release candidate

qiita_core/configuration_manager.py

@@ -145,6 +145,7 @@ def __init__(self):
         self._get_ebi(config)
         self._get_vamps(config)
         self._get_portal(config)
+        self._iframe(config)
 
     def _get_main(self, config):
         """Get the configuration of the main section"""
@@ -286,3 +287,6 @@ def _get_portal(self, config):
                 self.portal_dir = self.portal_dir[:-1]
         else:
             self.portal_dir = ""
+
+    def _iframe(self, config):
+        self.iframe_qiimp = config.get('iframe', 'QIIMP')

qiita_core/support_files/config_test.cfg

@@ -162,3 +162,10 @@ PORTAL_DIR =
 
 # Full path to portal styling config file
 PORTAL_FP =
+
+
+# ----------------------------- iframes settings ---------------------------
+[iframe]
+# The real world QIIMP will always need to be accessed with https because Qiita
+# runs on https too
+QIIMP = https://localhost:8898/

qiita_core/tests/test_configuration_manager.py

@@ -105,6 +105,9 @@ def test_init(self):
         self.assertEqual(obs.portal, "QIITA")
         self.assertEqual(obs.portal_dir, "/portal")
 
+        # iframe section
+        self.assertEqual(obs.iframe_qiimp, "https://localhost:8898/")
+
     def test_init_error(self):
         with open(self.conf_fp, 'w') as f:
             f.write("\n")
@@ -347,6 +350,10 @@ def test_get_portal(self):
 
 # Full path to portal styling config file
 PORTAL_FP = /tmp/portal.cfg
+
+# ----------------------------- iframes settings ---------------------------
+[iframe]
+QIIMP = https://localhost:8898/
 """
 
 if __name__ == '__main__':

qiita_db/artifact.py

@@ -788,6 +788,15 @@ def can_be_submitted_to_ebi(self):
             True if the artifact can be submitted to EBI. False otherwise.
         """
         with qdb.sql_connection.TRN:
+            # we should always return False if this artifact is not directly
+            # attached to the prep_template or is the second after. In other
+            # words has more that one processing step behind it
+            fine_to_send = []
+            fine_to_send.extend([pt.artifact for pt in self.prep_templates])
+            fine_to_send.extend([c for a in fine_to_send for c in a.children])
+            if self not in fine_to_send:
+                return False
+
             sql = """SELECT can_be_submitted_to_ebi
                      FROM qiita.artifact_type
                         JOIN qiita.artifact USING (artifact_type_id)

qiita_db/meta_util.py

@@ -267,6 +267,7 @@ def sizeof_fmt(value, position):
     plt.grid()
     ax = plt.gca()
     ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(sizeof_fmt))
+    plt.xticks(rotation=90)
     plt.xlabel('Date')
     plt.ylabel('Storage space per data type')
 

qiita_db/metadata_template/test/test_util.py

@@ -7,6 +7,8 @@
 # -----------------------------------------------------------------------------
 
 from six import StringIO
+from inspect import currentframe, getfile
+from os.path import dirname, abspath, join
 from unittest import TestCase, main
 import warnings
 
@@ -69,6 +71,29 @@ def test_load_template_to_dataframe(self):
         exp.index.name = 'sample_name'
         assert_frame_equal(obs, exp)
 
+    def test_load_template_to_dataframe_xlsx(self):
+        mfp = join(dirname(abspath(getfile(currentframe()))), 'support_files')
+
+        # test loading a qiimp file
+        fp = join(mfp, 'a_qiimp_wb.xlsx')
+        obs = qdb.metadata_template.util.load_template_to_dataframe(fp)
+        exp = pd.DataFrame.from_dict(EXP_QIIMP, dtype=str)
+        exp.index.name = 'sample_name'
+        assert_frame_equal(obs, exp)
+
+        # test loading an empty qiimp file
+        fp = join(mfp, 'empty_qiimp_wb.xlsx')
+        with self.assertRaises(ValueError) as error:
+            qdb.metadata_template.util.load_template_to_dataframe(fp)
+        self.assertEqual(str(error.exception), "The template is empty")
+
+        # test loading non qiimp file
+        fp = join(mfp, 'not_a_qiimp_wb.xlsx')
+        obs = qdb.metadata_template.util.load_template_to_dataframe(fp)
+        exp = pd.DataFrame.from_dict(EXP_NOT_QIIMP, dtype=str)
+        exp.index.name = 'sample_name'
+        assert_frame_equal(obs, exp)
+
     def test_load_template_to_dataframe_qiime_map(self):
         obs = qdb.metadata_template.util.load_template_to_dataframe(
             StringIO(QIIME_TUTORIAL_MAP_SUBSET), index='#SampleID')
@@ -844,5 +869,19 @@ def test_get_pgsql_reserved_words(self):
     '1.SKD8.640184\tCGTAGAGCTCTC\tANL\tTest Project\tNone\tEMP\tBBBB\tAAAA\t'
     'GTGCCAGCMGCCGCGGTAA\tILLUMINA\ts_G1_L001_sequences\tValue for sample 2\n')
 
+EXP_QIIMP = {
+    'asfaewf': {'sample': 'f', 'oijnmk': 'f'},
+    'pheno': {'sample': 'med', 'oijnmk': 'missing: not provided'},
+    'bawer': {'sample': 'a', 'oijnmk': 'b'},
+    'aelrjg': {'sample': 'asfe', 'oijnmk': 'asfs'}
+}
+
+EXP_NOT_QIIMP = {
+    'myownidea': {
+        'sample5': 'I skipped some',
+        'sample1': 'sampleoneinfo',
+        'sample2': 'sampletwoinfo'}
+}
+
 if __name__ == '__main__':
     main()

qiita_db/metadata_template/util.py

@@ -182,6 +182,8 @@ def load_template_to_dataframe(fn, index='sample_name'):
                      regex=True, inplace=True)
     # removing columns with empty values
     template.dropna(axis='columns', how='all', inplace=True)
+    if template.empty:
+        raise ValueError("The template is empty")
 
     initial_columns = set(template.columns)
 

qiita_db/study.py

@@ -823,6 +823,34 @@ def ebi_study_accession(self, value):
             qdb.sql_connection.TRN.add(sql, [value, self.id])
             qdb.sql_connection.TRN.execute()
 
+    def _ebi_submission_jobs(self):
+        """Helper code to avoid duplication"""
+        plugin = qdb.software.Software.from_name_and_version(
+            'Qiita', 'alpha')
+        cmd = plugin.get_command('submit_to_EBI')
+
+        sql = """SELECT processing_job_id,
+                    pj.command_parameters->>'artifact' as aid,
+                    processing_job_status, can_be_submitted_to_ebi,
+                    array_agg(ebi_run_accession)
+                 FROM qiita.processing_job pj
+                 LEFT JOIN qiita.processing_job_status
+                    USING (processing_job_status_id)
+                 LEFT JOIN qiita.artifact ON (
+                    artifact_id = (
+                        pj.command_parameters->>'artifact')::INT)
+                 LEFT JOIN qiita.ebi_run_accession era USING (artifact_id)
+                 LEFT JOIN qiita.artifact_type USING (artifact_type_id)
+                 WHERE pj.command_parameters->>'artifact' IN (
+                    SELECT artifact_id::text
+                    FROM qiita.study_artifact WHERE study_id = {0})
+                    AND pj.command_id = {1}
+                 GROUP BY processing_job_id, aid, processing_job_status,
+                    can_be_submitted_to_ebi""".format(self._id, cmd.id)
+        qdb.sql_connection.TRN.add(sql)
+
+        return qdb.sql_connection.TRN.execute_fetchindex()
+
     @property
     def ebi_submission_status(self):
         """The EBI submission status of this study
@@ -846,24 +874,11 @@ def ebi_submission_status(self):
             if self.ebi_study_accession:
                 status = 'submitted'
 
-            plugin = qdb.software.Software.from_name_and_version(
-                'Qiita', 'alpha')
-            cmd = plugin.get_command('submit_to_EBI')
-
-            sql = """SELECT processing_job_id, command_parameters->>'artifact',
-                        processing_job_status
-                     FROM qiita.processing_job
-                     LEFT JOIN qiita.processing_job_status
-                        USING (processing_job_status_id)
-                     WHERE command_parameters->>'artifact' IN (
-                        SELECT artifact_id::text
-                        FROM qiita.study_artifact
-                        WHERE study_id = {0}) AND command_id = {1}""".format(
-                            self._id, cmd.id)
-            qdb.sql_connection.TRN.add(sql)
             jobs = defaultdict(dict)
-            for info in qdb.sql_connection.TRN.execute_fetchindex():
-                jid, aid, js = info
+            for info in self._ebi_submission_jobs():
+                jid, aid, js, cbste, era = info
+                if not cbste or era != [None]:
+                    continue
                 jobs[js][aid] = jid
 
             if 'queued' in jobs or 'running' in jobs:

qiita_db/test/test_artifact.py

@@ -11,6 +11,7 @@
 from datetime import datetime
 from os import close, remove
 from os.path import exists, join, basename
+from shutil import copyfile
 from functools import partial
 from json import dumps
 
@@ -811,6 +812,11 @@ def test_create_root_analysis(self):
         qdb.artifact.Artifact.delete(obs.id)
 
     def test_create_processed(self):
+        # make a copy of files for the can_be_submitted_to_ebi tests
+        lcopy = self.fp3 + '.fna'
+        self._clean_up_files.append(lcopy)
+        copyfile(self.fp3, lcopy)
+
         exp_params = qdb.software.Parameters.from_default_params(
             qdb.software.DefaultParameters(1), {'input_data': 1})
         before = datetime.now()
@@ -842,6 +848,15 @@ def test_create_processed(self):
         self.assertFalse(exists(self.filepaths_processed[0][0]))
         self.assertIsNone(obs.analysis)
 
+        # let's create another demultiplexed on top of the previous one to
+        # test can_be_submitted_to_ebi
+        exp_params = qdb.software.Parameters.from_default_params(
+            qdb.software.DefaultParameters(1), {'input_data': obs.id})
+        new = qdb.artifact.Artifact.create(
+            [(lcopy, 4)], "Demultiplexed", parents=[obs],
+            processing_parameters=exp_params)
+        self.assertFalse(new.can_be_submitted_to_ebi)
+
     def test_create_copy_files(self):
         exp_params = qdb.software.Parameters.from_default_params(
             qdb.software.DefaultParameters(1), {'input_data': 1})

qiita_db/test/test_study.py

@@ -593,6 +593,27 @@ def test_ebi_study_accession_setter(self):
 
     def test_ebi_submission_status(self):
         self.assertEqual(self.study.ebi_submission_status, 'submitted')
+
+        # let's test that even with a failed job nothing changes
+        # add a failed job for an artifact (2) that can be submitted
+        user = qdb.user.User('test@foo.bar')
+        qp = qdb.software.Software.from_name_and_version('Qiita', 'alpha')
+        cmd = qp.get_command('submit_to_EBI')
+        params = qdb.software.Parameters.load(cmd, values_dict={
+            'artifact': 2, 'submission_type': 'ADD'})
+        job = qdb.processing_job.ProcessingJob.create(user, params, True)
+        job._set_error('Killed by Admin')
+        # and just to be careful add a failed job for an artifact (1) that
+        # cannot be submitted
+        qp = qdb.software.Software.from_name_and_version('Qiita', 'alpha')
+        cmd = qp.get_command('submit_to_EBI')
+        params = qdb.software.Parameters.load(cmd, values_dict={
+            'artifact': 1, 'submission_type': 'ADD'})
+        job = qdb.processing_job.ProcessingJob.create(user, params, True)
+        job._set_error('Killed by Admin')
+        # should still return submited
+        self.assertEqual(self.study.ebi_submission_status, 'submitted')
+
         new = qdb.study.Study.create(
             qdb.user.User('test@foo.bar'),
             'NOT Identification of the Microbiomes for Cannabis Soils 5',

qiita_db/util.py

@@ -51,6 +51,9 @@
 from os.path import join, basename, isdir, exists
 from os import walk, remove, listdir, makedirs, rename
 from shutil import move, rmtree, copy as shutil_copy
+from openpyxl import load_workbook
+from tempfile import mkstemp
+from csv import writer as csv_writer
 from json import dumps
 from datetime import datetime
 from itertools import chain
@@ -1724,12 +1727,45 @@ def _is_string_or_bytes(s):
 
 
 def _get_filehandle(filepath_or, *args, **kwargs):
-    """Open file if `filepath_or` looks like a string/unicode/bytes, else
+    """Open file if `filepath_or` looks like a string/unicode/bytes/Excel, else
     pass through.
+
+    Notes
+    -----
+    If Excel, the code will write a temporary txt file with the contents. Also,
+    it will check if the file is a Qiimp file or a regular Excel file.
     """
     if _is_string_or_bytes(filepath_or):
         if h5py.is_hdf5(filepath_or):
             fh, own_fh = h5py.File(filepath_or, *args, **kwargs), True
+        elif filepath_or.endswith('.xlsx'):
+            # due to extension, let's assume Excel file
+            wb = load_workbook(filename=filepath_or, data_only=True)
+            sheetnames = wb.sheetnames
+            # let's check if Qiimp, they must be in same order
+            first_cell_index = 0
+            is_qiimp_wb = False
+            if sheetnames == ["Metadata", "Validation", "Data Dictionary",
+                              "metadata_schema", "metadata_form",
+                              "Instructions"]:
+                first_cell_index = 1
+                is_qiimp_wb = True
+            first_sheet = wb[sheetnames[0]]
+            cell_range = range(first_cell_index, first_sheet.max_column)
+            _, fp = mkstemp(suffix='.txt')
+            with open(fp, 'w') as fh:
+                cfh = csv_writer(fh, delimiter='\t')
+                for r in first_sheet.rows:
+                    if is_qiimp_wb:
+                        # check contents of first column; if they are a zero
+                        # (not a valid QIIMP sample_id) or a "No more than
+                        # max samples" message, there are no more valid rows,
+                        # so don't examine any more rows.
+                        fcv = str(r[cell_range[0]].value)
+                        if fcv == "0" or fcv.startswith("No more than"):
+                            break
+                    cfh.writerow([r[x].value for x in cell_range])
+            fh, own_fh = open(fp, *args, **kwargs), True
         else:
             fh, own_fh = open(filepath_or, *args, **kwargs), True
     else:

qiita_pet/handlers/api_proxy/tests/test_sample_template.py

@@ -173,7 +173,7 @@ def test_get_sample_template_processing_status(self):
         self.assertEqual(obs_at, "danger")
         self.assertEqual(obs_am, "Some</br>error")
 
-    def test_sample_template_summary_get_req_no_template(self):
+    def test_sample_template_columns_get_req_no_template(self):
         # Test sample template not existing
         obs = sample_template_get_req(self.new_study.id, 'test@foo.bar')
         exp = {'status': 'error',

qiita_pet/handlers/auth_handlers.py

@@ -56,7 +56,7 @@ def post(self):
                            "click the following link to verify email address: "
                            "%s/auth/verify/%s?email=%s\n\nBy clicking you are "
                            "accepting our term and conditions: "
-                           "%s/static/qiita_data_terms_of_use.html"
+                           "%s/iframe/?iframe=qiita-terms"
                            % (url, info['user_verify_code'],
                               url_escape(username), url))
             except Exception:

qiita_pet/handlers/base_handlers.py

@@ -7,6 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from tornado.web import RequestHandler
+
 from qiita_db.logger import LogEntry
 from qiita_db.user import User
 from qiita_pet.util import convert_text_html
@@ -78,6 +79,16 @@ def get(self):
         self.render("index.html", message=msg, level=lvl)
 
 
+class IFrame(BaseHandler):
+    '''Open one of the IFrame pages'''
+    def get(self):
+        msg = self.get_argument('message', '')
+        msg = convert_text_html(msg)
+        lvl = self.get_argument('level', '')
+        iframe = self.get_argument('iframe', '')
+        self.render("iframe.html", iframe=iframe, message=msg, level=lvl)
+
+
 class MockupHandler(BaseHandler):
     def get(self):
         self.render("mockup.html")

qiita_pet/handlers/stats.py

@@ -62,7 +62,8 @@ def get(self):
                     number_studies=stats['number_studies'],
                     number_of_samples=stats['number_of_samples'],
                     num_users=stats['num_users'],
-                    lat_longs=eval(stats['lat_longs']),
+                    lat_longs=eval(
+                        stats['lat_longs']) if stats['lat_longs'] else [],
                     num_studies_ebi=stats['num_studies_ebi'],
                     num_samples_ebi=stats['num_samples_ebi'],
                     number_samples_ebi_prep=stats['number_samples_ebi_prep'],

qiita_pet/handlers/study_handlers/__init__.py

@@ -23,14 +23,14 @@
                        ArtifactAdminAJAX, ArtifactGetSamples, ArtifactGetInfo)
 from .sample_template import (
     SampleTemplateHandler, SampleTemplateOverviewHandler,
-    SampleTemplateSummaryHandler, SampleAJAX)
+    SampleTemplateColumnsHandler, SampleAJAX)
 
 __all__ = ['ListStudiesHandler', 'StudyApprovalList', 'ShareStudyAJAX',
            'StudyEditHandler', 'CreateStudyAJAX', 'EBISubmitHandler',
            'VAMPSHandler', 'SearchStudiesAJAX', 'ArtifactGraphAJAX',
            'ArtifactAdminAJAX', 'StudyIndexHandler', 'StudyBaseInfoAJAX',
            'SampleTemplateHandler', 'SampleTemplateOverviewHandler',
-           'SampleTemplateSummaryHandler',
+           'SampleTemplateColumnsHandler',
            'PrepTemplateAJAX', 'NewArtifactHandler', 'PrepFilesHandler',
            'ListCommandsHandler', 'ListOptionsHandler', 'SampleAJAX',
            'StudyDeleteAjax', 'NewPrepTemplateAjax',