WIP: Next steps

qiita_core/configuration_manager.py

@@ -78,16 +78,12 @@ class ConfigurationManager(object):
     smtp_email : str
         The email address that mail will be sent from when sending mail from
         the SMTP server
-    ebi_access_key : str
-        The access key issued by EBI for REST submissions
     ebi_seq_xfer_user : str
         The user to use when submitting to EBI
     ebi_seq_xfer_pass : str
         The password for the ebi_seq_xfer_user
     ebi_seq_xfer_url : str
         The URL of EBI's sequence portal site
-    ebi_skip_curl_cert : bool
-        Whether or not to skip the certificate check when curling the metadata
     ebi_center_name : str
         The name of the sequencing center to use when doing EBI submissions
     ebi_organization_prefix : str
@@ -213,14 +209,11 @@ def _get_smtp(self, config):
 
     def _get_ebi(self, config):
         sec_get = partial(config.get, 'ebi')
-        sec_getbool = partial(config.getboolean, 'ebi')
 
-        self.ebi_access_key = sec_get('EBI_ACCESS_KEY')
         self.ebi_seq_xfer_user = sec_get('EBI_SEQ_XFER_USER')
         self.ebi_seq_xfer_pass = sec_get('EBI_SEQ_XFER_PASS')
         self.ebi_seq_xfer_url = sec_get('EBI_SEQ_XFER_URL')
         self.ebi_dropbox_url = sec_get('EBI_DROPBOX_URL')
-        self.ebi_skip_curl_cert = sec_getbool('EBI_SKIP_CURL_CERT')
         self.ebi_center_name = sec_get('EBI_CENTER_NAME')
         self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX')
 

qiita_core/support_files/config_test.cfg

@@ -99,9 +99,6 @@ ADMIN_PASSWORD =
 
 # ----------------------------- EBI settings -----------------------------
 [ebi]
-# The access key issued by EBI for REST submissions
-EBI_ACCESS_KEY =
-
 # The user to use when submitting to EBI
 EBI_SEQ_XFER_USER = Webin-41528
 
@@ -117,12 +114,8 @@ EBI_SEQ_XFER_URL = webin.ebi.ac.uk
 # testing URL
 EBI_DROPBOX_URL = https://www-test.ebi.ac.uk/ena/submit/drop-box/submit/
 
-# Skip acertificate check when calling curl (necessary for using the EBI test
-# dropbox URL)
-EBI_SKIP_CURL_CERT = True
-
 # The name of the sequencing center to use when doing EBI submissions
-EBI_CENTER_NAME = CCME-COLORADO
+EBI_CENTER_NAME = qiita-test
 
 # This string (with an underscore) will be prefixed to your EBI submission and
 # study aliases

qiita_db/data.py

@@ -77,6 +77,7 @@
 # -----------------------------------------------------------------------------
 
 from __future__ import division
+from future.utils import viewitems
 from datetime import datetime
 from os.path import join
 from functools import partial
@@ -1039,6 +1040,41 @@ def ebi_run_accessions(self):
             accessions = TRN.execute_fetchflatten()
         return accessions
 
+    @ebi_run_accessions.setter
+    def ebi_run_accessions(self, value):
+        """Sets the EBI run accessions
+
+        Parameters
+        ----------
+        value : dict of {str: str}
+            The EBI run accessions, keyed by samples id
+
+        Raises
+        ------
+        QiitaDBError
+            If the preprocessed data already has ebi run accessions
+        """
+        with TRN:
+            sql = """SELECT EXISTS(
+                        SELECT *
+                        FROM qiita.ebi_run_accession
+                        WHERE preprocessed_data_id = %s
+                            AND ebi_run_accession IS NOT NULL)"""
+            TRN.add(sql, [self.id])
+            exists = TRN.execute_fetchlast()
+            if exists:
+                raise QiitaDBError(
+                    "Can't update EBI run accessions. The processed data %d "
+                    "already has ebi run accessions." % self.id)
+
+            sql = """INSERT INTO qiita.ebi_run_accession
+                        (preprocessed_data_id, sample_id, ebi_run_accession)
+                     VALUES (%s, %s, %s)"""
+            sql_params = [[self.id, sample, accession]
+                          for sample, accession in viewitems(value)]
+            TRN.add(sql, sql_params, many=True)
+            TRN.execute()
+
 
 class ProcessedData(BaseData):
     r"""Object for dealing with processed data

qiita_db/test/test_data.py

@@ -604,6 +604,43 @@ def test_ebi_run_accessions(self):
         self.assertEqual(PreprocessedData(1).ebi_run_accessions, exp,)
         self.assertEqual(PreprocessedData(2).ebi_run_accessions, [])
 
+    def test_ebi_run_accessions_setter(self):
+        new_vals = {
+            '1.SKB1.640202': 'ERR1000001',
+            '1.SKB2.640194': 'ERR1000002',
+            '1.SKB3.640195': 'ERR1000003',
+            '1.SKB4.640189': 'ERR1000004',
+            '1.SKB5.640181': 'ERR1000005',
+            '1.SKB6.640176': 'ERR1000006',
+            '1.SKB7.640196': 'ERR1000007',
+            '1.SKB8.640193': 'ERR1000008',
+            '1.SKB9.640200': 'ERR1000009',
+            '1.SKD1.640179': 'ERR1000010',
+            '1.SKD2.640178': 'ERR1000011',
+            '1.SKD3.640198': 'ERR1000012',
+            '1.SKD4.640185': 'ERR1000013',
+            '1.SKD5.640186': 'ERR1000014',
+            '1.SKD6.640190': 'ERR1000015',
+            '1.SKD7.640191': 'ERR1000016',
+            '1.SKD8.640184': 'ERR1000017',
+            '1.SKD9.640182': 'ERR1000018',
+            '1.SKM1.640183': 'ERR1000019',
+            '1.SKM2.640199': 'ERR1000020',
+            '1.SKM3.640197': 'ERR1000021',
+            '1.SKM4.640180': 'ERR1000022',
+            '1.SKM5.640177': 'ERR1000023',
+            '1.SKM6.640187': 'ERR1000024',
+            '1.SKM7.640188': 'ERR1000025',
+            '1.SKM8.640201': 'ERR1000026',
+            '1.SKM9.640192': 'ERR1000027'}
+        with self.assertRaises(QiitaDBError):
+            PreprocessedData(1).ebi_run_accessions = new_vals
+
+        ppd = PreprocessedData(2)
+        self.assertEqual(ppd.ebi_run_accessions, [])
+        ppd.ebi_run_accessions = new_vals
+        self.assertEqual(ppd.ebi_run_accessions, sorted(new_vals.values()))
+
 
 @qiita_test_checker()
 class ProcessedDataTests(TestCase):

qiita_pet/support_files/doc/source/tutorials/prepare-templates.rst

@@ -52,8 +52,6 @@ the centralized `Qiita server <http://qiita.microbio.me>`__:
 +===================================+======================================================================+=============================================================================================================================================================================================================================================+
 | ``sample_type``                   | free text                                                            | Description of the type of sample.                                                                                                                                                                                                          |
 +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``description``                   | free text                                                            | Description of the sample.                                                                                                                                                                                                                  |
-+-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``physical_specimen_remaining``   | ``y`` or ``n``                                                       | Is there still physical sample (e.g., soil, not DNA) available?                                                                                                                                                                             |
 +-----------------------------------+----------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``dna_extracted``                 | ``y`` or ``n``                                                       | Has DNA already been extracted for this sample?                                                                                                                                                                                             |
@@ -98,6 +96,8 @@ These are the columns required for successfully submit your data to EBI:
 +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``library_construction_protocol``   | free text                                 | Brief description or reference to the protocol that was used for preparing this amplicon library starting from DNA, usually this includes what genomic region was targeted such as *16S*, *ITS*, *18S*, etc. |
 +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| ``description``                     | free text                                 | Description of the sample.                                                                                                                                                                                   |
++-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``experiment_design_description``   | free text                                 | High-level description of the study (for example, *A longitudinal study of the gut microbiome of two human subjects*).                                                                                       |
 +-------------------------------------+-------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 

qiita_ware/commands.py

@@ -10,6 +10,10 @@
 from shutil import rmtree
 from tarfile import open as taropen
 from tempfile import mkdtemp
+from os import environ
+from traceback import format_exc
+from subprocess import call
+from shlex import split as shsplit
 
 from moi.job import system_call
 from qiita_db.study import Study
@@ -19,11 +23,9 @@
 from qiita_core.qiita_settings import qiita_config
 from qiita_ware.ebi import EBISubmission
 from qiita_ware.exceptions import ComputeError, EBISubmissionError
-from traceback import format_exc
-from os import environ
 
 
-def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
+def submit_EBI(preprocessed_data_id, action, send):
     """Submit a preprocessed data to EBI
 
     Parameters
@@ -34,27 +36,19 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
         The action to perform with this data
     send : bool
         True to actually send the files
-    fastq_dir_fp : str, optional
-        The fastq filepath
-
-    Notes
-    -----
-    If fastq_dir_fp is passed, it must not contain any empty files, or
-    gzipped empty files
     """
     # step 1: init and validate
     ebi_submission = EBISubmission(preprocessed_data_id, action)
 
     # step 2: generate demux fastq files
-    ebi_submission.preprocessed_data.update_insdc_status('demuxing samples')
+    ebi_submission.study.ebi_submission_status = 'submitting'
     try:
         ebi_submission.generate_demultiplexed_fastq()
     except:
         error_msg = format_exc()
-        if isdir(ebi_submission.ebi_dir):
-            rmtree(ebi_submission.ebi_dir)
-        ebi_submission.preprocessed_data.update_insdc_status(
-            'failed: %s' % error_msg)
+        if isdir(ebi_submission.full_ebi_dir):
+            rmtree(ebi_submission.full_ebi_dir)
+        ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
         LogEntry.create('Runtime', error_msg,
                         info={'ebi_submission': preprocessed_data_id})
         raise
@@ -66,34 +60,42 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
         # step 4: sending sequences
         old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
         environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
-        seqs_cmds = ebi_submission.generate_send_sequences_cmd()
+
         LogEntry.create('Runtime',
                         ("Submitting sequences for pre_processed_id: "
                          "%d" % preprocessed_data_id))
-        try:
-            # place holder for moi call see #1477
-            pass
-        except:
-            LogEntry.create('Fatal', seqs_cmds,
-                            info={'ebi_submission': preprocessed_data_id})
-        else:
-            LogEntry.create('Runtime',
-                            ('Submission of sequences of pre_processed_id: '
-                             '%d completed successfully' %
-                             preprocessed_data_id))
+        for cmd in ebi_submission.generate_send_sequences_cmd():
+            cmd_pieces = shsplit(cmd)
+            try:
+                call(cmd_pieces, stdout=open(ebi_submission.ascp_reply, 'a'))
+            except:
+                with open(ebi_submission.ascp_reply, 'r') as f:
+                    content = f.read()
+                ebi_submission.study.ebi_submission_status = (
+                    "failed: ASCP - %s" % content)
+                LogEntry.create('Fatal', content,
+                                info={
+                                    'ebi_submission': preprocessed_data_id,
+                                    'fail': cmd})
+        LogEntry.create('Runtime',
+                        ('Submission of sequences of pre_processed_id: '
+                         '%d completed successfully' %
+                         preprocessed_data_id))
         environ['ASPERA_SCP_PASS'] = old_ascp_pass
 
         # step 5: sending xml and parsing answer
-        xmls_cmds = ebi_submission.generate_curl_command()
+        xmls_cmds = shsplit(ebi_submission.generate_curl_command())
         LogEntry.create('Runtime',
                         ("Submitting XMLs for pre_processed_id: "
                          "%d" % preprocessed_data_id))
         try:
-            # place holder for moi call see #1477
-            xmls_cmds_moi = xmls_cmds
+            call(xmls_cmds, stdout=open(ebi_submission.curl_reply, 'w'))
+            with open(ebi_submission.curl_reply, 'r') as f:
+                xml_content = f.read()
         except:
-            # handle exception
-            LogEntry.create('Fatal', seqs_cmds,
+            with open(ebi_submission.curl_reply, 'r') as f:
+                xml_content = f.read()
+            LogEntry.create('Fatal', xml_content,
                             info={'ebi_submission': preprocessed_data_id})
         else:
             LogEntry.create('Runtime',
@@ -103,12 +105,13 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
 
         try:
             st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
-                ebi_submission.parse_EBI_reply(xmls_cmds_moi)
+                ebi_submission.parse_EBI_reply(xml_content)
         except EBISubmissionError as e:
             le = LogEntry.create(
-                'Fatal', "Command: %s\nError: %s\n" % (xmls_cmds_moi, str(e)),
+                'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
                 info={'ebi_submission': preprocessed_data_id})
-            ebi_submission.preprocessed_data.update_insdc_status('failed')
+            ebi_submission.study.ebi_submission_status = (
+                "failed: XML submission, log id: %d" % le.id)
             raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
 
         ebi_submission.study.ebi_submission_status = 'submitted'

qiita_ware/dispatchable.py

@@ -52,11 +52,7 @@ def preprocessor(study_id, prep_template_id, param_id, param_constructor):
 
 def submit_to_ebi(preprocessed_data_id, submission_type):
     """Submit a study to EBI"""
-    study_acc, submission_acc = submit_EBI(preprocessed_data_id,
-                                           submission_type,
-                                           True)
-
-    return study_acc, submission_acc
+    submit_EBI(preprocessed_data_id, submission_type, True)
 
 
 def submit_to_VAMPS(preprocessed_data_id):