Fix qiita all tests

qiita_db/metadata_template/base_metadata_template.py

@@ -1041,7 +1041,7 @@ def add_filepath(self, filepath, conn_handler=None, fp_id=None):
         fp_id = self._fp_id if fp_id is None else fp_id
 
         try:
-            fpp_id = insert_filepaths([(filepath, self._fp_id)], None,
+            fpp_id = insert_filepaths([(filepath, fp_id)], None,
                                       "templates", "filepath", conn_handler,
                                       move_files=False)[0]
             values = (self._id, fpp_id)

qiita_db/metadata_template/prep_template.py

@@ -421,6 +421,13 @@ def create_qiime_mapping_file(self):
             'description': 'Description',
         }
 
+        if 'reverselinkerprimer' in self.categories():
+            rename_cols['reverselinkerprimer'] = 'ReverseLinkerPrimer'
+            new_cols = ['BarcodeSequence', 'LinkerPrimerSequence',
+                        'ReverseLinkerPrimer']
+        else:
+            new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
+
         # getting the latest sample template
         conn_handler = SQLConnectionHandler()
         sql = """SELECT filepath_id, filepath
@@ -474,7 +481,6 @@ def create_qiime_mapping_file(self):
         cols.remove('BarcodeSequence')
         cols.remove('LinkerPrimerSequence')
         cols.remove('Description')
-        new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
         new_cols.extend(cols)
         new_cols.append('Description')
         mapping = mapping[new_cols]
@@ -524,3 +530,25 @@ def status(self):
         pd_statuses = conn_handler.execute_fetchall(sql, (self._id,))
 
         return infer_status(pd_statuses)
+
+    @property
+    def qiime_map_fp(self):
+        """The QIIME mapping filepath attached to the prep template
+
+        Returns
+        -------
+        str
+            The filepath of the QIIME mapping file
+        """
+        conn_handler = SQLConnectionHandler()
+
+        sql = """SELECT filepath_id, filepath
+                 FROM qiita.filepath
+                    JOIN qiita.{0} USING (filepath_id)
+                    JOIN qiita.filepath_type USING (filepath_type_id)
+                 WHERE {1} = %s AND filepath_type = 'qiime_map'
+                 ORDER BY filepath_id DESC""".format(self._filepath_table,
+                                                     self._id_column)
+        fn = conn_handler.execute_fetchall(sql, (self._id,))[0][1]
+        base_dir = get_mountpoint('templates')[0][1]
+        return join(base_dir, fn)

qiita_db/metadata_template/test/test_prep_template.py

@@ -1293,6 +1293,12 @@ def test_update_category(self):
         self.assertEqual(self.tester['1.SKB8.640193']['center_name'], 'FOO')
         self.assertEqual(self.tester['1.SKD8.640184']['center_name'], 'BAR')
 
+    def test_qiime_map_fp(self):
+        pt = PrepTemplate(1)
+        exp = join(get_mountpoint('templates')[0][1],
+                   '1_prep_1_qiime_19700101-000000.txt')
+        self.assertEqual(pt.qiime_map_fp, exp)
+
 
 EXP_PREP_TEMPLATE = (
     'sample_name\tbarcode\tcenter_name\tcenter_project_name\t'

qiita_db/test/test_analysis.py

@@ -182,9 +182,9 @@ def test_retrieve_dropped_samples(self):
             "lab_person_id": StudyPerson(1)
         }
         metadata_dict = {
-            'SKB8.640193': {'physical_location': 'location1',
-                            'has_physical_specimen': True,
-                            'has_extracted_data': True,
+            'SKB8.640193': {'physical_specimen_location': 'location1',
+                            'physical_specimen_remaining': True,
+                            'dna_extracted': True,
                             'sample_type': 'type1',
                             'required_sample_info_status': 'received',
                             'collection_timestamp':
@@ -194,9 +194,9 @@ def test_retrieve_dropped_samples(self):
                             'str_column': 'Value for sample 1',
                             'latitude': 42.42,
                             'longitude': 41.41},
-            'SKD8.640184': {'physical_location': 'location1',
-                            'has_physical_specimen': True,
-                            'has_extracted_data': True,
+            'SKD8.640184': {'physical_specimen_location': 'location1',
+                            'physical_specimen_remaining': True,
+                            'dna_extracted': True,
                             'sample_type': 'type1',
                             'required_sample_info_status': 'received',
                             'collection_timestamp':
@@ -206,9 +206,9 @@ def test_retrieve_dropped_samples(self):
                             'str_column': 'Value for sample 2',
                             'latitude': 4.2,
                             'longitude': 1.1},
-            'SKB7.640196': {'physical_location': 'location1',
-                            'has_physical_specimen': True,
-                            'has_extracted_data': True,
+            'SKB7.640196': {'physical_specimen_location': 'location1',
+                            'physical_specimen_remaining': True,
+                            'dna_extracted': True,
                             'sample_type': 'type1',
                             'required_sample_info_status': 'received',
                             'collection_timestamp':

qiita_db/test/test_commands.py

@@ -723,8 +723,9 @@ def test_update_preprocessed_data_from_cmd_ppd(self):
 
 SAMPLE_TEMPLATE = (
     "sample_name\trequired_sample_info_status\tcollection_timestamp\t"
-    "sample_type\thas_physical_specimen\tphysical_location\thas_extracted_data"
-    "\thost_subject_id\tTreatment\tDOB\tlatitude\tlongitude\tDescription\n"
+    "sample_type\tphysical_specimen_remaining\tphysical_specimen_location\t"
+    "dna_extracted\thost_subject_id\tTreatment\tDOB\tlatitude\tlongitude"
+    "\tDescription\n"
     "PC.354\treceived\t2014-06-18 16:44\ttype_1\tTrue\tLocation_1\tTrue\t"
     "HS_ID_PC.354\tControl\t20061218\t1.88401499993\t56.0003871552\t"
     "Control_mouse_I.D._354\n"
@@ -739,8 +740,8 @@ def test_update_preprocessed_data_from_cmd_ppd(self):
     "Fasting_mouse_I.D._636")
 
 PREP_TEMPLATE = (
-    'sample_name\tbarcodesequence\tcenter_name\tcenter_project_name\t'
-    'description\tebi_submission_accession\temp_status\tlinkerprimersequence\t'
+    'sample_name\tbarcode\tcenter_name\tcenter_project_name\t'
+    'description\tebi_submission_accession\temp_status\tprimer\t'
     'run_prefix\tstr_column\tplatform\tlibrary_construction_protocol\t'
     'experiment_design_description\n'
     'SKB7.640196\tCCTCTGAGAGCT\tANL\tTest Project\tskb7\tNone\tEMP\t'

qiita_ware/processing_pipeline.py

@@ -40,33 +40,44 @@ def _get_qiime_minimal_mapping(prep_template, out_dir):
         The paths to the qiime minimal mapping files
     """
     from functools import partial
+    from collections import defaultdict
     from os.path import join
     import pandas as pd
 
-    # Get the data in a pandas DataFrame, so it is easier to manage
-    pt = prep_template.to_dataframe()
+    # The prep templates has a QIIME mapping file, get it
+    qiime_map = pd.read_csv(prep_template.qiime_map_fp, sep='\t',
+                            keep_default_na=False, na_values=['unknown'],
+                            index_col=False,
+                            converters=defaultdict(lambda: str))
+    qiime_map.set_index('#SampleID', inplace=True, drop=True)
 
-    # We now need to rename some columns to be QIIME compliant.
-    # Hopefully, this conversion won't be needed if QIIME relaxes its
-    # constraints
-    pt.rename(columns={'barcodesequence': 'BarcodeSequence',
-                       'linkerprimersequence': 'LinkerPrimerSequence'},
-              inplace=True)
-    pt['Description'] = pd.Series(['Qiita MMF'] * len(pt.index),
-                                  index=pt.index)
+    # We use our own description to avoid potential processing problems
+    qiime_map['Description'] = pd.Series(['Qiita MMF'] * len(qiime_map.index),
+                                         index=qiime_map.index)
 
     # We ensure the order of the columns as QIIME is expecting
-    cols = ['BarcodeSequence', 'LinkerPrimerSequence', 'Description']
+    if 'ReverseLinkerPrimer' in qiime_map:
+        cols = ['BarcodeSequence', 'LinkerPrimerSequence',
+                'ReverseLinkerPrimer', 'Description']
+    else:
+        cols = ['BarcodeSequence', 'LinkerPrimerSequence', 'Description']
 
-    # If the study has more than 1 lane, we should generate a qiita MMF for
-    # each of the lanes. We know how to split the prep template based on
-    # the run_prefix column
-    output_fps = []
     path_builder = partial(join, out_dir)
-    for prefix, df in pt.groupby('run_prefix'):
-        df = df[cols]
-        out_fp = path_builder("%s_MMF.txt" % prefix)
-        output_fps.append(out_fp)
+    if 'run_prefix' in qiime_map:
+        # The study potentially has more than 1 lane, so we should generate a
+        # qiime MMF for each of the lanes. We know how to split the prep
+        # template based on the run_prefix column
+        output_fps = []
+        for prefix, df in qiime_map.groupby('run_prefix'):
+            df = df[cols]
+            out_fp = path_builder("%s_MMF.txt" % prefix)
+            output_fps.append(out_fp)
+            df.to_csv(out_fp, index_label="#SampleID", sep='\t')
+    else:
+        # The study only has one lane, just write the MMF
+        df = qiime_map[cols]
+        out_fp = path_builder("prep_%d_MMF.txt" % prep_template.id)
+        output_fps = [out_fp]
         df.to_csv(out_fp, index_label="#SampleID", sep='\t')
 
     return output_fps