Fix 2321

qiita_db/study.py

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 from __future__ import division
+from collections import defaultdict
 from future.utils import viewitems
 from copy import deepcopy
 from itertools import chain
@@ -43,9 +44,6 @@
 import qiita_db as qdb
 
 
-_VALID_EBI_STATUS = ('not submitted', 'submitting', 'submitted')
-
-
 class Study(qdb.base.QiitaObject):
     r"""Study object to access to the Qiita Study information
 
@@ -81,8 +79,7 @@ class Study(qdb.base.QiitaObject):
     _table = "study"
     _portal_table = "study_portal"
     # The following columns are considered not part of the study info
-    _non_info = frozenset(["email", "study_title", "ebi_submission_status",
-                           "ebi_study_accession"])
+    _non_info = frozenset(["email", "study_title", "ebi_study_accession"])
 
     def _lock_non_sandbox(self):
         """Raises QiitaDBStatusError if study is non-sandboxed"""
@@ -212,10 +209,19 @@ def get_info(cls, study_ids=None, info_cols=None):
                 args.append(tuple(study_ids))
 
             qdb.sql_connection.TRN.add(sql, args)
-            res = qdb.sql_connection.TRN.execute_fetchindex()
-            if study_ids is not None and len(res) != len(study_ids):
+            rows = qdb.sql_connection.TRN.execute_fetchindex()
+            if study_ids is not None and len(rows) != len(study_ids):
                 raise qdb.exceptions.QiitaDBError(
                     'Non-portal-accessible studies asked for!')
+
+            res = []
+            for r in rows:
+                r = dict(r)
+                if 'ebi_study_accession' in info_cols:
+                    r['ebi_submission_status'] = cls(
+                        r['study_id']).ebi_submission_status
+                res.append(r)
+
             return res
 
     @classmethod
@@ -825,39 +831,58 @@ def ebi_submission_status(self):
         -------
         str
             The study EBI submission status
-        """
-        with qdb.sql_connection.TRN:
-            sql = """SELECT ebi_submission_status
-                     FROM qiita.{0}
-                     WHERE study_id = %s""".format(self._table)
-            qdb.sql_connection.TRN.add(sql, [self.id])
-            return qdb.sql_connection.TRN.execute_fetchlast()
 
-    @ebi_submission_status.setter
-    def ebi_submission_status(self, value):
-        """Sets the study's EBI submission status
-
-        Parameters
-        ----------
-        value : str {%s}
-            The new EBI submission status
-
-        Raises
-        ------
-        ValueError
-            If the status is not known
+        Notes
+        -----
+        There are 4 possible states: 'not submitted', 'submitting',
+        'submitted' & 'failed'. We are going to assume 'not submitted' if the
+        study doesn't have an accession, 'submitted' if it has an accession,
+        'submitting' if there are submit_to_EBI jobs running using the study
+        artifacts, & 'failed' if there are artifacts with failed jobs without
+        successful ones.
         """
-        if not (value in _VALID_EBI_STATUS or
-                value.startswith('failed')):
-            raise ValueError("Unknown status: %s" % value)
+        status = 'not submitted'
         with qdb.sql_connection.TRN:
-            sql = """UPDATE qiita.{}
-                     SET ebi_submission_status = %s
-                     WHERE study_id = %s""".format(self._table)
-            qdb.sql_connection.TRN.add(sql, [value, self.id])
-            qdb.sql_connection.TRN.execute()
-
-    ebi_submission_status.__doc__.format(', '.join(_VALID_EBI_STATUS))
+            if self.ebi_study_accession:
+                status = 'submitted'
+
+            plugin = qdb.software.Software.from_name_and_version(
+                'Qiita', 'alpha')
+            cmd = plugin.get_command('submit_to_EBI')
+
+            sql = """SELECT processing_job_id, command_parameters->>'artifact',
+                        processing_job_status
+                     FROM qiita.processing_job
+                     LEFT JOIN qiita.processing_job_status
+                        USING (processing_job_status_id)
+                     WHERE command_parameters->>'artifact' IN (
+                        SELECT artifact_id::text
+                        FROM qiita.study_artifact
+                        WHERE study_id = {0}) AND command_id = {1}""".format(
+                            self._id, cmd.id)
+            qdb.sql_connection.TRN.add(sql)
+            jobs = defaultdict(dict)
+            for info in qdb.sql_connection.TRN.execute_fetchindex():
+                jid, aid, js = info
+                jobs[js][aid] = jid
+
+            if 'queued' in jobs or 'running' in jobs:
+                status = 'submitting'
+            elif 'error' in jobs:
+                aids_error = []
+                aids_other = []
+                for s, aids in jobs.items():
+                    for aid in aids.keys():
+                        if s == 'error':
+                            aids_error.append(aid)
+                        else:
+                            aids_other.append(aid)
+                difference = set(aids_error) - set(aids_other)
+                if difference:
+                    status = ('Some artifact submissions failed: %s' %
+                              ', '.join(map(str, list(difference))))
+
+        return status
 
     @property
     def tags(self):

qiita_db/support_files/patches/64.sql

@@ -0,0 +1,4 @@
+-- March 7, 2018
+-- delete ebi_submission_status from study
+
+ALTER TABLE qiita.study DROP ebi_submission_status;

qiita_db/support_files/qiita-db.dbs

@@ -1370,9 +1370,6 @@ Controlled Vocabulary]]></comment>
 			<column name="study_abstract" type="text" jt="12" mandatory="y" />
 			<column name="vamps_id" type="varchar" jt="12" />
 			<column name="ebi_study_accession" type="varchar" jt="12" />
-			<column name="ebi_submission_status" type="varchar" jt="12" mandatory="y" >
-				<defo>&#039;not_submitted&#039;</defo>
-			</column>
 			<index name="pk_study" unique="PRIMARY_KEY" >
 				<column name="study_id" />
 			</index>

qiita_db/support_files/qiita-db.html

@@ -1590,7 +1590,6 @@
   <use id='nn' x='1787' y='537' xlink:href='#nn'/><a xlink:href='#study.study_abstract'><text x='1803' y='547'>study_abstract</text><title>study_abstract text not null</title></a>
   <a xlink:href='#study.vamps_id'><text x='1803' y='562'>vamps_id</text><title>vamps_id varchar</title></a>
   <a xlink:href='#study.ebi_study_accession'><text x='1803' y='577'>ebi_study_accession</text><title>ebi_study_accession varchar</title></a>
-  <use id='nn' x='1787' y='582' xlink:href='#nn'/><a xlink:href='#study.ebi_submission_status'><text x='1803' y='592'>ebi_submission_status</text><title>ebi_submission_status varchar not null default &#039;not&#095;submitted&#039;</title></a>
 
 <!-- ============= Table 'per_study_tags' ============= -->
 <rect class='table' x='2145' y='443' width='105' height='75' rx='7' ry='7' />
@@ -4434,12 +4433,6 @@
 		<td> varchar   </td>
 		<td>  </td>
 	</tr>
-	<tr>
-		<td>*</td>
-		<td><a name='study.ebi_submission_status'>ebi&#095;submission&#095;status</a></td>
-		<td> varchar   DEFO 'not_submitted' </td>
-		<td>  </td>
-	</tr>
 <tr><th colspan='4'><b>Indexes</b></th></tr>
 	<tr>		<td>Pk</td><td>pk&#095;study</td>
 		<td> ON study&#095;id</td>

qiita_db/test/test_study.py

@@ -253,16 +253,15 @@ def test_get_info(self):
                     'publications', 'study_title']
         obs = qdb.study.Study.get_info([1], exp_keys)
         self.assertEqual(len(obs), 1)
-        obs = dict(obs[0])
-        exp = {
+        exp = [{
             'metadata_complete': True, 'reprocess': False,
             'timeseries_type': 'None',
             'publications': [{'f1': '10.100/123456', 'f2': True},
                              {'f1': '123456', 'f2': False},
                              {'f1': '10.100/7891011', 'f2': True},
                              {'f1': '7891011', 'f2': False}],
             'study_title': 'Identification of the Microbiomes for Cannabis '
-            'Soils'}
+            'Soils'}]
         self.assertEqual(obs, exp)
 
         # Test get specific keys for all studies
@@ -279,13 +278,18 @@ def test_get_info(self):
 
         s = qdb.study.Study.create(user, 'test_study_1', info=info)
         obs = qdb.study.Study.get_info(info_cols=exp_keys)
-        exp = [[True, [{'f1': '7891011', 'f2': False},
-                       {'f1': '10.100/7891011', 'f2': True},
-                       {'f1': '123456', 'f2': False},
-                       {'f1': '10.100/123456', 'f2': True}], False,
-                'Identification of the Microbiomes for Cannabis Soils',
-                'None'],
-               [False, None, False, 'test_study_1', 'None']]
+        exp = [
+            {'metadata_complete': True, 'reprocess': False,
+             'timeseries_type': 'None',
+             'publications': [{'f1': '7891011', 'f2': False},
+                              {'f1': '10.100/7891011', 'f2': True},
+                              {'f1': '123456', 'f2': False},
+                              {'f1': '10.100/123456', 'f2': True}],
+             'study_title': 'Identification of the Microbiomes '
+                            'for Cannabis Soils'},
+            {'metadata_complete': False, 'reprocess': False,
+             'timeseries_type': 'None', 'publications': None,
+             'study_title': 'test_study_1'}]
         self.assertEqual(obs, exp)
         qdb.study.Study.delete(s.id)
 
@@ -596,23 +600,6 @@ def test_ebi_submission_status(self):
         self.assertEqual(new.ebi_submission_status, 'not submitted')
         qdb.study.Study.delete(new.id)
 
-    def test_ebi_submission_status_setter(self):
-        new = qdb.study.Study.create(
-            qdb.user.User('test@foo.bar'), 'Test 1', self.info)
-        self.assertEqual(new.ebi_submission_status, "not submitted")
-        new.ebi_submission_status = 'submitting'
-        self.assertEqual(new.ebi_submission_status, 'submitting')
-        new.ebi_submission_status = 'failed: something horrible happened'
-        self.assertEqual(new.ebi_submission_status,
-                         'failed: something horrible happened')
-        new.ebi_submission_status = 'submitted'
-        self.assertEqual(new.ebi_submission_status, 'submitted')
-
-        with self.assertRaises(ValueError):
-            new.ebi_submission_status = "unknown"
-
-        qdb.study.Study.delete(new.id)
-
     def test_set_info(self):
         """Set info in a study"""
         newinfo = {

qiita_db/util.py

@@ -1327,7 +1327,7 @@ def generate_study_list(study_ids, public_only=False):
     The main select might look scary but it's pretty simple:
     - We select the requiered fields from qiita.study and qiita.study_person
         SELECT metadata_complete, study_abstract, study_id, study_alias,
-            study_title, ebi_study_accession, ebi_submission_status,
+            study_title, ebi_study_accession,
             qiita.study_person.name AS pi_name,
             qiita.study_person.email AS pi_email,
     - the total number of samples collected by counting sample_ids
@@ -1363,7 +1363,7 @@ def generate_study_list(study_ids, public_only=False):
     with qdb.sql_connection.TRN:
         sql = """
             SELECT metadata_complete, study_abstract, study_id, study_alias,
-                study_title, ebi_study_accession, ebi_submission_status,
+                study_title, ebi_study_accession,
                 qiita.study_person.name AS pi_name,
                 qiita.study_person.email AS pi_email,
                 (SELECT COUNT(sample_id) FROM qiita.study_sample
@@ -1428,7 +1428,9 @@ def generate_study_list(study_ids, public_only=False):
             del info["shared_with_name"]
             del info["shared_with_email"]
 
-            info['status'] = qdb.study.Study(info['study_id']).status
+            study = qdb.study.Study(info['study_id'])
+            info['status'] = study.status
+            info['ebi_submission_status'] = study.ebi_submission_status
             infolist.append(info)
     return infolist
 
@@ -1453,7 +1455,7 @@ def generate_study_list_without_artifacts(study_ids, public_only=False):
     The main select might look scary but it's pretty simple:
     - We select the requiered fields from qiita.study and qiita.study_person
         SELECT metadata_complete, study_abstract, study_id, study_alias,
-            study_title, ebi_study_accession, ebi_submission_status,
+            study_title, ebi_study_accession,
             qiita.study_person.name AS pi_name,
             qiita.study_person.email AS pi_email,
     - the total number of samples collected by counting sample_ids
@@ -1468,7 +1470,7 @@ def generate_study_list_without_artifacts(study_ids, public_only=False):
     with qdb.sql_connection.TRN:
         sql = """
             SELECT metadata_complete, study_abstract, study_id, study_alias,
-                study_title, ebi_study_accession, ebi_submission_status,
+                study_title, ebi_study_accession,
                 qiita.study_person.name AS pi_name,
                 qiita.study_person.email AS pi_email,
                 (SELECT COUNT(sample_id) FROM qiita.study_sample
@@ -1506,7 +1508,9 @@ def generate_study_list_without_artifacts(study_ids, public_only=False):
             del info["pi_email"]
             del info["pi_name"]
 
-            info['status'] = qdb.study.Study(info['study_id']).status
+            study = qdb.study.Study(info['study_id'])
+            info['status'] = study.status
+            info['ebi_submission_status'] = study.ebi_submission_status
             infolist.append(info)
     return infolist
 

qiita_ware/commands.py

@@ -39,14 +39,12 @@ def submit_EBI(artifact_id, action, send, test=False):
     ebi_submission = EBISubmission(artifact_id, action)
 
     # step 2: generate demux fastq files
-    ebi_submission.study.ebi_submission_status = 'submitting'
     try:
         ebi_submission.generate_demultiplexed_fastq()
     except Exception:
         error_msg = format_exc()
         if isdir(ebi_submission.full_ebi_dir):
             rmtree(ebi_submission.full_ebi_dir)
-        ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
         LogEntry.create('Runtime', error_msg,
                         info={'ebi_submission': artifact_id})
         raise
@@ -104,11 +102,8 @@ def submit_EBI(artifact_id, action, send, test=False):
             le = LogEntry.create(
                 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
                 info={'ebi_submission': artifact_id})
-            ebi_submission.study.ebi_submission_status = (
-                "failed: XML parsing, log id: %d" % le.id)
             raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
 
-        ebi_submission.study.ebi_submission_status = 'submitted'
         if action == 'ADD':
             if st_acc:
                 ebi_submission.study.ebi_study_accession = st_acc

qiita_ware/test/test_commands.py

@@ -7,7 +7,8 @@
 #
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
-from unittest import TestCase, main
+from unittest import TestCase, main, skipIf
+from os import environ
 from os.path import join
 from tempfile import mkdtemp
 import pandas as pd
@@ -144,14 +145,13 @@ def test_submit_EBI_parse_EBI_reply_failure(self):
         with self.assertRaises(ComputeError):
             submit_EBI(ppd.id, 'VALIDATE', True)
 
+    @skipIf(
+        environ.get('ASPERA_SCP_PASS', '') == '', 'skip: ascp not configured')
     def test_full_submission(self):
         artifact = self.generate_new_study_with_preprocessed_data()
-
-        # just making sure
-        self.assertEqual(artifact.study.ebi_submission_status, 'not submitted')
-
+        self.assertEqual(
+            artifact.study.ebi_submission_status, 'not submitted')
         submit_EBI(artifact.id, 'VALIDATE', True, test=True)
-
         self.assertEqual(artifact.study.ebi_submission_status, 'submitted')