Merge relax-md-req into master

qiita_db/metadata_template/base_metadata_template.py

@@ -540,6 +540,13 @@ def _clean_validate_template(cls, md_template, study_id, restriction_dict):
         -------
         md_template : DataFrame
             Cleaned copy of the input md_template
+
+        Raises
+        ------
+        QiitaDBColumnError
+            If the sample names in md_template contains invalid names
+        QiitaDBWarning
+            If there are missing columns required for some functionality
         """
         cls._check_subclass()
         invalid_ids = get_invalid_sample_names(md_template.index)
@@ -988,7 +995,8 @@ def to_file(self, fp, samples=None):
         if samples is not None:
             df = df.loc[samples]
 
-        # Apply some sorting to the dataframe
+        # Sorting the dataframe so multiple serializations of the metadata
+        # template are consistent.
         df.sort_index(axis=0, inplace=True)
         df.sort_index(axis=1, inplace=True)
 
@@ -1021,7 +1029,7 @@ def to_dataframe(self):
 
         return df
 
-    def add_filepath(self, filepath, conn_handler=None):
+    def add_filepath(self, filepath, conn_handler=None, fp_id=None):
         r"""Populates the DB tables for storing the filepath and connects the
         `self` objects with this filepath"""
         # Check that this function has been called from a subclass
@@ -1030,6 +1038,7 @@ def add_filepath(self, filepath, conn_handler=None):
         # Check if the connection handler has been provided. Create a new
         # one if not.
         conn_handler = conn_handler if conn_handler else SQLConnectionHandler()
+        fp_id = self._fp_id if fp_id is None else fp_id
 
         try:
             fpp_id = insert_filepaths([(filepath, self._fp_id)], None,

qiita_db/metadata_template/constants.py

@@ -13,10 +13,12 @@
 
 # A dict containing the restrictions that apply to the sample templates
 SAMPLE_TEMPLATE_COLUMNS = {
+    # The following columns are required by EBI for submission
     'EBI': Restriction(columns={'collection_timestamp': 'timestamp',
                                 'physical_specimen_location': 'varchar'},
                        error_msg="EBI submission disabled"),
-    'Qiita_main': Restriction(columns={'sample_type': 'varchar',
+    # The following columns are required for the official main QIITA site
+    'qiita_main': Restriction(columns={'sample_type': 'varchar',
                                        'description': 'varchar',
                                        'physical_specimen_remaining': 'bool',
                                        'dna_extracted': 'bool',
@@ -26,8 +28,9 @@
                               error_msg="Processed data approval disabled")
 }
 
-# A dict containing the restrictions that apply to the sample templates
+# A dict containing the restrictions that apply to the prep templates
 PREP_TEMPLATE_COLUMNS = {
+    # The following columns are required by EBI for submission
     'EBI': Restriction(
         columns={'primer': 'varchar',
                  'center_name': 'varchar',
@@ -43,12 +46,15 @@
 TARGET_GENE_DATA_TYPES = ['16S', '18S', 'ITS']
 
 PREP_TEMPLATE_COLUMNS_TARGET_GENE = {
-    'Demultiplex': Restriction(
+    # The following columns are required by QIIME to execute split libraries
+    'demultiplex': Restriction(
         columns={'barcode': 'varchar',
                  'primer': 'varchar'},
         error_msg="Demultiplexing disabled. You will not be able to "
                   "preprocess your raw data"),
-    'Demultiplex_multiple': Restriction(
+    # The following columns are required by Qiita to know how to execute split
+    # libraries using QIIME over a study with multiple illumina lanes
+    'demultiplex_multiple': Restriction(
         columns={'barcode': 'varchar',
                  'primer': 'varchar',
                  'run_prefix': 'varchar'},

qiita_db/metadata_template/prep_template.py

@@ -11,12 +11,14 @@
 from os.path import join
 from time import strftime
 from copy import deepcopy
+import warnings
 
 import pandas as pd
 
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
 from qiita_db.exceptions import (QiitaDBColumnError, QiitaDBUnknownIDError,
-                                 QiitaDBError, QiitaDBExecutionError)
+                                 QiitaDBError, QiitaDBExecutionError,
+                                 QiitaDBWarning)
 from qiita_db.sql_connection import SQLConnectionHandler
 from qiita_db.ontology import Ontology
 from qiita_db.util import (convert_to_id,
@@ -127,7 +129,7 @@ def create(cls, md_template, raw_data, study, data_type,
             pt_cols.update(PREP_TEMPLATE_COLUMNS_TARGET_GENE)
 
         md_template = cls._clean_validate_template(md_template, study.id,
-                                                   PREP_TEMPLATE_COLUMNS)
+                                                   pt_cols)
 
         # Insert the metadata template
         # We need the prep_id for multiple calls below, which currently is not
@@ -402,12 +404,20 @@ def create_qiime_mapping_file(self):
         ------
         ValueError
             If the prep template is not a subset of the sample template
+        QiitaDBWarning
+            If the QIIME-required columns are not present in the template
+
+        Notes
+        -----
+        We cannot ensure that the QIIME-required columns are present in the
+        metadata map. However, we have to generate a QIIME-compliant mapping
+        file. Since the user may need a QIIME mapping file, but not these
+        QIIME-required columns, we are going to create them and
+        populate them with the value XXQIITAXX.
         """
         rename_cols = {
             'barcode': 'BarcodeSequence',
-            'barcodesequence': 'BarcodeSequence',
             'primer': 'LinkerPrimerSequence',
-            'linkerprimersequence': 'LinkerPrimerSequence',
             'description': 'Description',
         }
 
@@ -434,23 +444,30 @@ def create_qiime_mapping_file(self):
         if not pt_sample_names.issubset(st_sample_names):
             raise ValueError(
                 "Prep template is not a sub set of the sample template, files"
-                "%s - samples: %s" % (sample_template_fp,
-                                      str(pt_sample_names-st_sample_names)))
+                "%s - samples: %s"
+                % (sample_template_fp,
+                   ', '.join(pt_sample_names-st_sample_names)))
 
         mapping = pt.join(st, lsuffix="_prep")
         mapping.rename(columns=rename_cols, inplace=True)
 
-        # We cannot ensure that the QIIME-required columns are present in the
-        # metadata map. However, we have to generate a QIIME-compliant mapping
-        # file. Since the user may need a QIIME mapping file, but not these
-        # QIIME-required columns, we are going to create them here and
-        # populate them with the value XXQIITAXX
+        # Pre-populate the QIIME-required columns with the value XXQIITAXX
         index = mapping.index
         placeholder = ['XXQIITAXX'] * len(index)
+        missing = []
         for val in viewvalues(rename_cols):
             if val not in mapping:
+                missing.append(val)
                 mapping[val] = pd.Series(placeholder, index=index)
 
+        if missing:
+            warnings.warn(
+                "Some columns required to generate a QIIME-compliant mapping "
+                "file are not present in the template. A placeholder value "
+                "(XXQIITAXX) has been used to populate these columns. Missing "
+                "columns: %s" % ', '.join(missing),
+                QiitaDBWarning)
+
         # Gets the orginal mapping columns and readjust the order to comply
         # with QIIME requirements
         cols = mapping.columns.values.tolist()
@@ -472,7 +489,9 @@ def create_qiime_mapping_file(self):
                        sep='\t')
 
         # adding the fp to the object
-        self.add_filepath(filepath)
+        self.add_filepath(
+            filepath, conn_handler=conn_handler,
+            fp_id=convert_to_id("qiime_map", "filepath_type"))
 
         return filepath