Merge branch 'master' of github.com:qiime/QiiTa into cart-branch

Author: josenavas

qiita_db/data.py

@@ -103,7 +103,7 @@ class BaseData(QiitaObject):
     --------
     RawData
     PreprocessedData
-    PreprocessedData
+    ProcessedData
     """
     _filepath_table = "filepath"
 
@@ -807,6 +807,73 @@ def create(cls, study, preprocessed_params_table, preprocessed_params_id,
         ppd.add_filepaths(filepaths, conn_handler)
         return ppd
 
+    @classmethod
+    def delete(cls, ppd_id):
+        """Removes the preprocessed data with id ppd_id
+
+        Parameters
+        ----------
+        ppd_id : int
+            The preprocessed data id
+
+        Raises
+        ------
+        QiitaDBStatusError
+            If the preprocessed data status is not sandbox or if the
+            preprocessed data EBI and VAMPS submission is not in a valid state
+            ['not submitted', 'failed']
+        QiitaDBError
+            If the preprocessed data has been processed
+        """
+        valid_submission_states = ['not submitted', 'failed']
+        ppd = cls(ppd_id)
+        if ppd.status != 'sandbox':
+            raise QiitaDBStatusError(
+                "Illegal operation on non sandboxed preprocessed data")
+        elif ppd.submitted_to_vamps_status() not in valid_submission_states:
+            raise QiitaDBStatusError(
+                "Illegal operation. This preprocessed data has or is being "
+                "added to VAMPS.")
+        elif ppd.submitted_to_insdc_status() not in valid_submission_states:
+            raise QiitaDBStatusError(
+                "Illegal operation. This preprocessed data has or is being "
+                "added to EBI.")
+
+        conn_handler = SQLConnectionHandler()
+
+        processed_data = [str(n[0]) for n in conn_handler.execute_fetchall(
+            "SELECT processed_data_id FROM qiita.preprocessed_processed_data "
+            "WHERE preprocessed_data_id = {0} ORDER BY "
+            "processed_data_id".format(ppd_id))]
+
+        if processed_data:
+            raise QiitaDBError(
+                "Preprocessed data %d cannot be removed because it was used "
+                "to generate the following processed data: %s" % (
+                    ppd_id, ', '.join(processed_data)))
+
+        # delete
+        queue = "delete_preprocessed_data_%d" % ppd_id
+        conn_handler.create_queue(queue)
+
+        sql = ("DELETE FROM qiita.prep_template_preprocessed_data WHERE "
+               "preprocessed_data_id = {0}".format(ppd_id))
+        conn_handler.add_to_queue(queue, sql)
+
+        sql = ("DELETE FROM qiita.preprocessed_filepath WHERE "
+               "preprocessed_data_id = {0}".format(ppd_id))
+        conn_handler.add_to_queue(queue, sql)
+
+        sql = ("DELETE FROM qiita.study_preprocessed_data WHERE "
+               "preprocessed_data_id = {0}".format(ppd_id))
+        conn_handler.add_to_queue(queue, sql)
+
+        sql = ("DELETE FROM qiita.preprocessed_data WHERE "
+               "preprocessed_data_id = {0}".format(ppd_id))
+        conn_handler.add_to_queue(queue, sql)
+
+        conn_handler.execute_queue(queue)
+
     @property
     def processed_data(self):
         r"""The processed data list generated from this preprocessed data"""
@@ -1275,7 +1342,7 @@ def delete(cls, processed_data_id):
         """
         if cls(processed_data_id).status != 'sandbox':
             raise QiitaDBStatusError(
-                "Illegal operation on non sandbox processed data")
+                "Illegal operation on non sandboxed processed data")
 
         conn_handler = SQLConnectionHandler()
 

qiita_db/metadata_template/base_metadata_template.py

@@ -542,7 +542,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id,
         headers = list(md_template.keys())
 
         # Get the required columns from the DB
-        db_cols = get_table_cols(cls._table, conn_handler)
+        db_cols = sorted(get_table_cols(cls._table, conn_handler))
         # Remove the sample_id and _id_column columns
         db_cols.remove('sample_id')
         db_cols.remove(cls._id_column)
@@ -561,7 +561,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id,
             values, many=True)
 
         # Insert rows on *_columns table
-        headers = list(set(headers).difference(db_cols))
+        headers = sorted(set(headers).difference(db_cols))
         datatypes = get_datatypes(md_template.ix[:, headers])
         # psycopg2 requires a list of tuples, in which each tuple is a set
         # of values to use in the string formatting of the query. We have all

qiita_db/metadata_template/sample_template.py

@@ -481,32 +481,6 @@ def update(self, md_template):
                     if '_qiime_' not in basename(fp):
                         pt.create_qiime_mapping_file(fp)
 
-    def remove_category(self, category):
-        """Remove a category from the sample template
-
-        Parameters
-        ----------
-        category : str
-            The category to remove
-
-        Raises
-        ------
-        QiitaDBColumnError
-            If the column does not exist in the table
-        """
-        table_name = self._table_name(self.study_id)
-        conn_handler = SQLConnectionHandler()
-
-        if category not in self.categories():
-            raise QiitaDBColumnError("Column %s does not exist in %s" %
-                                     (category, table_name))
-
-        # This operation may invalidate another user's perspective on the
-        # table
-        conn_handler.execute("""
-            ALTER TABLE qiita.{0} DROP COLUMN {1}""".format(table_name,
-                                                            category))
-
     def update_category(self, category, samples_and_values):
         """Update an existing column
 

qiita_db/metadata_template/test/test_prep_template.py

@@ -582,39 +582,40 @@ def test_add_common_creation_steps_to_queue(self):
             'VALUES (%s, %s, %s)')
 
         sql_create_table = (
-            'CREATE TABLE qiita.prep_2 '
-            '(sample_id varchar NOT NULL, str_column varchar, '
-            'run_prefix varchar, barcodesequence varchar, platform varchar, '
-            'linkerprimersequence varchar, '
-            'experiment_design_description varchar, '
-            'library_construction_protocol varchar)')
+            'CREATE TABLE qiita.prep_2 (sample_id varchar NOT NULL, '
+            'barcodesequence varchar, experiment_design_description varchar, '
+            'library_construction_protocol varchar, '
+            'linkerprimersequence varchar, platform varchar, '
+            'run_prefix varchar, str_column varchar)')
 
         sql_insert_dynamic = (
             'INSERT INTO qiita.prep_2 '
-            '(sample_id, str_column, run_prefix, barcodesequence, platform, '
-            'linkerprimersequence, experiment_design_description, '
-            'library_construction_protocol) '
+            '(sample_id, barcodesequence, experiment_design_description, '
+            'library_construction_protocol, linkerprimersequence, platform, '
+            'run_prefix, str_column) '
             'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)')
 
         sql_insert_dynamic_params_1 = (
-            '2.SKB8.640193', 'Value for sample 1', 's_G1_L001_sequences',
-            'GTCCGCAAGTTA', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA')
+            '2.SKB8.640193', 'GTCCGCAAGTTA', 'BBBB', 'AAAA',
+            'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+            'Value for sample 1')
         sql_insert_dynamic_params_2 = (
-            '2.SKD8.640184', 'Value for sample 2', 's_G1_L001_sequences',
-            'CGTAGAGCTCTC', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA')
+            '2.SKD8.640184', 'CGTAGAGCTCTC', 'BBBB', 'AAAA',
+            'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+            'Value for sample 2')
 
         exp = [
             (sql_insert_common, sql_insert_common_params_1),
             (sql_insert_common, sql_insert_common_params_2),
-            (sql_insert_prep_columns, (2, 'str_column', 'varchar')),
-            (sql_insert_prep_columns, (2, 'run_prefix', 'varchar')),
             (sql_insert_prep_columns, (2, 'barcodesequence', 'varchar')),
-            (sql_insert_prep_columns, (2, 'platform', 'varchar')),
-            (sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')),
             (sql_insert_prep_columns,
                 (2, 'experiment_design_description', 'varchar')),
             (sql_insert_prep_columns,
                 (2, 'library_construction_protocol', 'varchar')),
+            (sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')),
+            (sql_insert_prep_columns, (2, 'platform', 'varchar')),
+            (sql_insert_prep_columns, (2, 'run_prefix', 'varchar')),
+            (sql_insert_prep_columns, (2, 'str_column', 'varchar')),
             (sql_create_table, None),
             (sql_insert_dynamic, sql_insert_dynamic_params_1),
             (sql_insert_dynamic, sql_insert_dynamic_params_2)]
@@ -794,17 +795,20 @@ def test_create(self):
         # The new table hosts the correct values
         obs = self.conn_handler.execute_fetchall(
             "SELECT * FROM qiita.prep_2")
-        # sample_id, study_id, str_column, ebi_submission_accession,
-        # run_prefix, barcodesequence, linkerprimersequence
-        exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
-                's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
-                's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
-                's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]
+
+        # barcodesequence, ebi_submission_accession,
+        # experiment_design_description, library_construction_protocol,
+        # linkerprimersequence, platform, run_prefix, str_column
+        exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
+                'Value for sample 3'],
+               ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 1'],
+               ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 2']]
+
         self.assertEqual(sorted(obs), sorted(exp))
 
         # prep and qiime files have been created
@@ -858,17 +862,18 @@ def test_create_already_prefixed_samples(self):
         # The new table hosts the correct values
         obs = self.conn_handler.execute_fetchall(
             "SELECT * FROM qiita.prep_2")
-        # sample_id, study_id, str_column, ebi_submission_accession,
-        # run_prefix, barcodesequence, linkerprimersequence
-        exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
-                's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
-                's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
-                's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]
+        # barcodesequence, ebi_submission_accession,
+        # experiment_design_description, library_construction_protocol,
+        # linkerprimersequence, platform, run_prefix, str_column
+        exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
+                'Value for sample 3'],
+               ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 1'],
+               ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 2']]
         self.assertEqual(sorted(obs), sorted(exp))
 
         # prep and qiime files have been created
@@ -959,17 +964,18 @@ def test_create_data_type_id(self):
         # The new table hosts the correct values
         obs = self.conn_handler.execute_fetchall(
             "SELECT * FROM qiita.prep_2")
-        # sample_id, str_column, ebi_submission_accession,
-        # run_prefix, barcodesequence, linkerprimersequence
-        exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
-                's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
-                's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
-               ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
-                's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
-                'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]
+        # barcodesequence, ebi_submission_accession,
+        # experiment_design_description, library_construction_protocol,
+        # linkerprimersequence, platform, run_prefix, str_column
+        exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
+                'Value for sample 3'],
+               ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 1'],
+               ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
+                'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
+                'Value for sample 2']]
         self.assertEqual(sorted(obs), sorted(exp))
 
     def test_create_error(self):

qiita_db/metadata_template/test/test_sample_template.py

@@ -802,23 +802,20 @@ def test_add_common_creation_steps_to_queue(self):
 
         sql_insert_required = (
             'INSERT INTO qiita.required_sample_info '
-            '(study_id, sample_id, physical_location, has_physical_specimen, '
-            'has_extracted_data, sample_type, required_sample_info_status_id, '
-            'collection_timestamp, host_subject_id, description, latitude, '
-            'longitude) '
+            '(study_id, sample_id, collection_timestamp, description, '
+            'has_extracted_data, has_physical_specimen, host_subject_id, '
+            'latitude, longitude, physical_location, '
+            'required_sample_info_status_id, sample_type) '
             'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)')
         sql_insert_required_params_1 = (
-            2, '2.Sample1', 'location1', True, True, 'type1', 1,
-            datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
-            'Test Sample 1', 42.42, 41.41)
+            2, '2.Sample1', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 1',
+            True, True, 'NotIdentified', 42.42, 41.41, 'location1', 1, 'type1')
         sql_insert_required_params_2 = (
-            2, '2.Sample2', 'location1', True, True, 'type1', 1,
-            datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
-            'Test Sample 2', 4.2, 1.1)
+            2, '2.Sample2', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 2',
+            True, True, 'NotIdentified', 4.2, 1.1, 'location1', 1, 'type1')
         sql_insert_required_params_3 = (
-            2, '2.Sample3', 'location1', True, True, 'type1', 1,
-            datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
-            'Test Sample 3', 4.8, 4.41)
+            2, '2.Sample3', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 3',
+            True, True, 'NotIdentified', 4.8, 4.41, 'location1', 1, 'type1')
 
         sql_insert_sample_cols = (
             'INSERT INTO qiita.study_sample_columns '
@@ -1237,18 +1234,6 @@ def test_add_category(self):
         obs = {k: v['new_column'] for k, v in self.tester.items()}
         self.assertEqual(obs, exp)
 
-    def test_remove_category(self):
-        with self.assertRaises(QiitaDBColumnError):
-            self.tester.remove_category('does not exist')
-
-        for v in self.tester.values():
-            self.assertIn('elevation', v)
-
-        self.tester.remove_category('elevation')
-
-        for v in self.tester.values():
-            self.assertNotIn('elevation', v)
-
     def test_to_file(self):
         """to file writes a tab delimited file with all the metadata"""
         fd, fp = mkstemp()

qiita_db/test/test_data.py

@@ -478,6 +478,48 @@ def test_create_data_type_only(self):
         # preprocessed_data_id, filepath_id
         self.assertEqual(obs, [[3, obs_id - 1], [3, obs_id]])
 
+    def test_delete_basic(self):
+        """Correctly deletes a preprocessed data"""
+        # testing regular delete
+        ppd = PreprocessedData.create(
+            self.study, self.params_table,
+            self.params_id, self.filepaths, prep_template=self.prep_template,
+            ebi_submission_accession=self.ebi_submission_accession,
+            ebi_study_accession=self.ebi_study_accession)
+        PreprocessedData.delete(ppd.id)
+
+        # testing that the deleted preprocessed data can't be instantiated
+        with self.assertRaises(QiitaDBUnknownIDError):
+            PreprocessedData(ppd.id)
+        # and for completeness testing that it raises an error if ID
+        # doesn't exist
+        with self.assertRaises(QiitaDBUnknownIDError):
+            PreprocessedData.delete(ppd.id)
+
+        # testing that we can not remove cause the preprocessed data != sandbox
+        with self.assertRaises(QiitaDBStatusError):
+            PreprocessedData.delete(1)
+
+    def test_delete_advanced(self):
+        # testing that we can not remove cause preprocessed data has been
+        # submitted to EBI or VAMPS
+        ppd = PreprocessedData.create(
+            self.study, self.params_table,
+            self.params_id, self.filepaths, prep_template=self.prep_template,
+            ebi_submission_accession=self.ebi_submission_accession,
+            ebi_study_accession=self.ebi_study_accession)
+
+        # fails due to VAMPS submission
+        ppd.update_vamps_status('success')
+        with self.assertRaises(QiitaDBStatusError):
+            PreprocessedData.delete(ppd.id)
+        ppd.update_vamps_status('failed')
+
+        # fails due to EBI submission
+        ppd.update_insdc_status('success', 'AAAA', 'AAAA')
+        with self.assertRaises(QiitaDBStatusError):
+            PreprocessedData.delete(ppd.id)
+
     def test_create_error_dynamic_table(self):
         """Raises an error if the preprocessed_params_table does not exist"""
         with self.assertRaises(IncompetentQiitaDeveloperError):