WIP: EBI-ENA test

.travis.yml

@@ -4,11 +4,11 @@ sudo: false
 env:
   global:
     - PYTHON_VERSION=3.5
+    - secure: "Xjuqz8/I6DYHq2wF0ZjLo0JxBUMNAmiK201PtLAGdqD8dgmcr/CdyU5On3Ok+PEcdro151TK6gTNYLpxOa5IkLnuwPkaSjXXl4Et/IgRwQHUR0VHFZ3MXUzbeJeCgAMCeVwYUfjVnP3On9Al7di2GK9h60hJwBiwZ6APAE60U+Y="
   matrix:
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
     - TEST_ADD_STUDIES=True COVER_PACKAGE="qiita_core qiita_ware"
-    - secure: ta4qrBQleTSwDUml9+jtnJES1qHCgLtjUpOHKX25ZCdw3vi/SpXxyErn5yrXvdefIDYtspVv3OWpWesiwFf3Qq1PSWJBeIrxdYizqebRr6GuUcdPN+Mj8PQ09UDWY3GthMfb26oIKmWi9YyELjwqkb6zztSW/e276scrEWppsVI=
 before_install:
   - redis-server --version
   - redis-server --port 7777 &
@@ -20,9 +20,6 @@ before_install:
   - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh
   - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh
   - ./ascp-install-3.5.4.102989-linux-64-qiita.sh
-  # once we have ebi testing we should uncomment this line
-  # look for EBI below as it's part of this
-  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_a2e23aea5f14_key -iv $encrypted_a2e23aea5f14_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d ; fi
 install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
@@ -65,10 +62,6 @@ install:
   - source deactivate
   - source activate qiita
 before_script:
-  # EBI, see the end of before_install about why this block is commented out
-  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then
-  #   export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
-  #   fi
   - qiita-env make --no-load-ontologies
   - qiita-test-install
 script:

qiita_pet/support_files/doc/source/gettingstartedguide/index.rst

@@ -126,13 +126,6 @@ Required fields for EBI submission
 
 In order to submit your data to EBI, certain fields are required in your sample information file. You can download a spread sheet containing these required fields on the `Knight Lab website <https://knightlab.ucsd.edu/wordpress/?page_id=478>`__ under "MetaData Template".
 
-Checking for Errors In Sample information
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* `Keemei <https://keemei.qiime2.org/>`__ is a Google Sheets add-on for validating sample metadata. Validation of sample metadata is important before beginning any analysis. Install Keemei and follow the instructions on its website to validate your sample information spread sheet.
-
-**Remember that spelling and capitalization matters in Qiita**
-
 Attaching the Sample Information to the Study
 ---------------------------------------------
 
@@ -318,12 +311,6 @@ additional column is required.
 |                  |             | and ``my-data.fastq`` becomes ``my-data``).                                                |
 +------------------+-------------+--------------------------------------------------------------------------------------------+
 
-Checking for Errors In Preparation information
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* `Keemei <https://keemei.qiime2.org/>`__ is a Google Sheets add-on for validating sample metadata. Validation of sample metadata is important before beginning any analysis. Install Keemei and follow the instructions on its website to validate your sample information spread sheet.
-
-**Remember that spelling and capitalization matters in Qiita**
 
 Attaching Preparation Information to the Study
 ----------------------------------------------

qiita_ware/commands.py

@@ -21,20 +21,22 @@
 from qiita_ware.exceptions import ComputeError, EBISubmissionError
 
 
-def submit_EBI(preprocessed_data_id, action, send):
-    """Submit a preprocessed data to EBI
+def submit_EBI(artifact_id, action, send, test=False):
+    """Submit a artifact to EBI
 
     Parameters
     ----------
-    preprocessed_data_id : int
-        The preprocesssed data id
+    artifact_id : int
+        The artifact id
     action : %s
         The action to perform with this data
     send : bool
         True to actually send the files
+    test : bool
+        If True some restrictions will be ignored, only used in parse_EBI_reply
     """
     # step 1: init and validate
-    ebi_submission = EBISubmission(preprocessed_data_id, action)
+    ebi_submission = EBISubmission(artifact_id, action)
 
     # step 2: generate demux fastq files
     ebi_submission.study.ebi_submission_status = 'submitting'
@@ -46,7 +48,7 @@ def submit_EBI(preprocessed_data_id, action, send):
             rmtree(ebi_submission.full_ebi_dir)
         ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
         LogEntry.create('Runtime', error_msg,
-                        info={'ebi_submission': preprocessed_data_id})
+                        info={'ebi_submission': artifact_id})
         raise
 
     # step 3: generate and write xml files
@@ -56,11 +58,12 @@ def submit_EBI(preprocessed_data_id, action, send):
         # step 4: sending sequences
         if action != 'MODIFY':
             old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
-            environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
+            if old_ascp_pass == '':
+                environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
 
             LogEntry.create('Runtime',
                             ("Submitting sequences for pre_processed_id: "
-                             "%d" % preprocessed_data_id))
+                             "%d" % artifact_id))
             for cmd in ebi_submission.generate_send_sequences_cmd():
                 stdout, stderr, rv = system_call(cmd)
                 if rv != 0:
@@ -69,17 +72,19 @@ def submit_EBI(preprocessed_data_id, action, send):
                     raise ComputeError(error_msg)
                 open(ebi_submission.ascp_reply, 'a').write(
                     'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
+
+            ascp_passwd = environ['ASPERA_SCP_PASS']
             environ['ASPERA_SCP_PASS'] = old_ascp_pass
             LogEntry.create('Runtime',
                             ('Submission of sequences of pre_processed_id: '
-                             '%d completed successfully' %
-                             preprocessed_data_id))
+                             '%d completed successfully' % artifact_id))
 
         # step 5: sending xml and parsing answer
-        xmls_cmds = ebi_submission.generate_curl_command()
+        xmls_cmds = ebi_submission.generate_curl_command(
+            ebi_seq_xfer_pass=ascp_passwd)
         LogEntry.create('Runtime',
                         ("Submitting XMLs for pre_processed_id: "
-                         "%d" % preprocessed_data_id))
+                         "%d" % artifact_id))
         xml_content, stderr, rv = system_call(xmls_cmds)
         if rv != 0:
             error_msg = ("Error:\nStd output:%s\nStd error:%s" % (
@@ -88,18 +93,17 @@ def submit_EBI(preprocessed_data_id, action, send):
         else:
             LogEntry.create('Runtime',
                             ('Submission of sequences of pre_processed_id: '
-                             '%d completed successfully' %
-                             preprocessed_data_id))
+                             '%d completed successfully' % artifact_id))
         open(ebi_submission.curl_reply, 'w').write(
             'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
 
         try:
             st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
-                ebi_submission.parse_EBI_reply(xml_content)
+                ebi_submission.parse_EBI_reply(xml_content, test=test)
         except EBISubmissionError as e:
             le = LogEntry.create(
                 'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
-                info={'ebi_submission': preprocessed_data_id})
+                info={'ebi_submission': artifact_id})
             ebi_submission.study.ebi_submission_status = (
                 "failed: XML parsing, log id: %d" % le.id)
             raise ComputeError("EBI Submission failed! Log id: %d" % le.id)

qiita_ware/ebi.py

@@ -7,7 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from os.path import basename, join, isdir, isfile, exists
-from shutil import copyfile
+from shutil import copyfile, rmtree
 from os import makedirs, remove, listdir
 from datetime import date, timedelta
 from urllib import quote
@@ -835,13 +835,15 @@ def generate_send_sequences_cmd(self):
 
         return ascp_commands
 
-    def parse_EBI_reply(self, curl_result):
+    def parse_EBI_reply(self, curl_result, test=False):
         """Parse and verify reply from EBI after sending XML files
 
         Parameters
         ----------
         curl_result : str
             The reply sent by EBI after sending XML files
+        test : bool
+            If true we will assume is a test and ignore some parsing errors
 
         Returns
         -------
@@ -901,7 +903,10 @@ def parse_EBI_reply(self, curl_result):
             sample_id = self._sample_aliases[alias]
             sample_accessions[sample_id] = elem.get('accession')
             ext_id = elem.find('EXT_ID')
-            biosample_accessions[sample_id] = ext_id.get('accession')
+            if test:
+                biosample_accessions[sample_id] = 'test_' + sample_id
+            else:
+                biosample_accessions[sample_id] = ext_id.get('accession')
 
         def data_retriever(key, trans_dict):
             res = {}
@@ -1028,6 +1033,10 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None):
         dir_not_exists = not isdir(self.full_ebi_dir)
         missing_samples = []
         if dir_not_exists or rewrite_fastq:
+            # if it exists, remove folder and start from scratch
+            if isdir(self.full_ebi_dir):
+                rmtree(self.full_ebi_dir)
+
             makedirs(self.full_ebi_dir)
 
             if self.artifact.artifact_type == 'per_sample_FASTQ':

qiita_ware/test/test_commands.py

@@ -16,7 +16,7 @@
 from h5py import File
 from qiita_files.demux import to_hdf5
 
-from qiita_ware.exceptions import ComputeError
+from qiita_ware.exceptions import ComputeError, EBISubmissionError
 from qiita_ware.commands import submit_EBI
 from qiita_db.study import Study, StudyPerson
 from qiita_db.software import DefaultParameters, Parameters
@@ -134,24 +134,25 @@ def generate_new_study_with_preprocessed_data(self):
         return ppd
 
     def test_submit_EBI_step_2_failure(self):
-        # see issue #406
-        # ppd = self.write_demux_files(PrepTemplate(1), False)
-        #
-        # with self.assertRaises(AttributeError):
-        #     submit_EBI(ppd.id, 'ADD', True)
-        pass
+        ppd = self.write_demux_files(PrepTemplate(1), False)
+
+        with self.assertRaises(EBISubmissionError):
+            submit_EBI(ppd.id, 'VALIDATE', True)
 
     def test_submit_EBI_parse_EBI_reply_failure(self):
         ppd = self.write_demux_files(PrepTemplate(1))
         with self.assertRaises(ComputeError):
-            submit_EBI(ppd.id, 'ADD', True)
+            submit_EBI(ppd.id, 'VALIDATE', True)
 
     def test_full_submission(self):
-        # see issue #406
-        # ppd = self.generate_new_study_with_preprocessed_data()
-        #
-        # submit_EBI(ppd.id, 'ADD', True)
-        pass
+        artifact = self.generate_new_study_with_preprocessed_data()
+
+        # just making sure
+        self.assertEqual(artifact.study.ebi_submission_status, 'not submitted')
+
+        submit_EBI(artifact.id, 'VALIDATE', True, test=True)
+
+        self.assertEqual(artifact.study.ebi_submission_status, 'submitted')
 
 
 FASTA_EXAMPLE = """>1.SKB2.640194_1 X orig_bc=X new_bc=X bc_diffs=0

qiita_ware/test/test_ebi.py

@@ -623,7 +623,7 @@ def test_generate_demultiplexed_fastq_failure(self):
         ebi_submission = EBISubmission(artifact.id, 'ADD')
         self.files_to_remove.append(ebi_submission.full_ebi_dir)
         with self.assertRaises(EBISubmissionError):
-            ebi_submission.generate_demultiplexed_fastq()
+            ebi_submission.generate_demultiplexed_fastq(rewrite_fastq=True)
 
         artifact = self.write_demux_files(PrepTemplate(1), 'WRONG-SEQS')
         ebi_submission = EBISubmission(artifact.id, 'ADD')
@@ -642,7 +642,11 @@ def test_generate_demultiplexed_fastq(self):
         # This is testing that only the samples with sequences are going to
         # be created
         ebi_submission = EBISubmission(artifact.id, 'ADD')
-        obs_demux_samples = ebi_submission.generate_demultiplexed_fastq()
+        # adding rewrite_fastq=True as it's possible to have duplicated ids
+        # and this will assure to get the right test
+        obs_demux_samples = ebi_submission.generate_demultiplexed_fastq(
+            rewrite_fastq=True)
+
         self.files_to_remove.append(ebi_submission.full_ebi_dir)
         self.assertItemsEqual(obs_demux_samples, exp_demux_samples)
         # testing that the samples/samples_prep and demux_samples are the same

scripts/qiita

@@ -187,16 +187,16 @@ def load_prep_template(fp, study, data_type):
 # #############################################################################
 
 @ebi.command()
-@click.option('--preprocessed_data_id', required=True, type=int)
+@click.option('--artifact-id', required=True, type=int)
 @click.option('--action', type=click.Choice(EBISubmission.valid_ebi_actions),
               default='submit', help='The generated XML files will specify '
               'this "action", which controls how the EBI servers handle the '
               'metadata')
 @click.option('--send/--no-send', default=False, help="Controls whether or "
               "not sequence files and metadata will actually be sent to EBI "
               "(default is to generate all the files, but not to send)")
-def submit(preprocessed_data_id, action, send):
-    _submit_EBI(preprocessed_data_id, action, send)
+def submit(artifact_id, action, send):
+    _submit_EBI(artifact_id, action, send)
 
 
 # #############################################################################