Merge branch 'relax-md-req' of https://github.com/biocore/qiita into relax-md-req

Author: josenavas

qiita_db/metadata_template/base_metadata_template.py

@@ -194,30 +194,13 @@ def _get_categories(self, conn_handler):
         set of str
             The set of all available metadata categories
         """
-        # Get all the required columns
-        required_cols = get_table_cols(self._table, conn_handler)
-        # Get all the the columns in the dynamic table
-        dynamic_cols = get_table_cols(self._dynamic_table, conn_handler)
-        # Get the union of the two previous lists
-        cols = set(required_cols).union(dynamic_cols)
-        # Remove the sample_id column and the study_id/raw_data_id columns,
-        # as this columns are used internally for data storage and they don't
-        # actually belong to the metadata
+        # Get all the columns
+        cols = get_table_cols(self._dynamic_table, conn_handler)
+        # Remove the sample_id column as this column is used internally for
+        # data storage and it doesn't actually belong to the metadata
         cols.remove('sample_id')
-        cols.remove(self._id_column)
-        try:
-            # study_id could be potentially removed by _id_column, so wrap
-            # in a try except
-            cols.remove('study_id')
-        except KeyError:
-            pass
-        # Change the *_id columns, as this is for convenience internally,
-        # and add the original categories
-        for key, value in viewitems(self._md_template.translate_cols_dict):
-            cols.remove(key)
-            cols.add(value)
 
-        return cols
+        return set(cols)
 
     def _to_dict(self):
         r"""Returns the categories and their values in a dictionary
@@ -229,22 +212,13 @@ def _to_dict(self):
         """
         conn_handler = SQLConnectionHandler()
         d = dict(conn_handler.execute_fetchone(
-            "SELECT * FROM qiita.{0} WHERE {1}=%s AND "
-            "sample_id=%s".format(self._table, self._id_column),
-            (self._md_template.id, self._id)))
-        dynamic_d = dict(conn_handler.execute_fetchone(
             "SELECT * from qiita.{0} WHERE "
             "sample_id=%s".format(self._dynamic_table),
             (self._id, )))
-        d.update(dynamic_d)
+
+        # Remove the sample_id, is not part of the metadata
         del d['sample_id']
-        del d[self._id_column]
-        d.pop('study_id', None)
 
-        # Modify all the *_id columns to include the string instead of the id
-        for k, v in viewitems(self._md_template.translate_cols_dict):
-            d[v] = self._md_template.str_cols_handlers[k][d[k]]
-            del d[k]
         return d
 
     def __len__(self):
@@ -283,39 +257,17 @@ def __getitem__(self, key):
         """
         conn_handler = SQLConnectionHandler()
         key = key.lower()
-        if key in self._get_categories(conn_handler):
-            # It's possible that the key is asking for one of the *_id columns
-            # that we have to do the translation
-            def handler(x):
-                return x
-
-            # prevent flake8 from complaining about the function not being
-            # used and a redefinition happening in the next few lines
-            handler(None)
-
-            if key in self._md_template.translate_cols_dict.values():
-                handler = (
-                    lambda x: self._md_template.str_cols_handlers[key][x])
-                key = "%s_id" % key
-            # Check if we have either to query the table with required columns
-            # or the dynamic table
-            if key in get_table_cols(self._table, conn_handler):
-                result = conn_handler.execute_fetchone(
-                    "SELECT {0} FROM qiita.{1} WHERE {2}=%s AND "
-                    "sample_id=%s".format(key, self._table, self._id_column),
-                    (self._md_template.id, self._id))[0]
-                return handler(result)
-            else:
-                return conn_handler.execute_fetchone(
-                    "SELECT {0} FROM qiita.{1} WHERE "
-                    "sample_id=%s".format(key, self._dynamic_table),
-                    (self._id, ))[0]
-        else:
+        if key not in self._get_categories(conn_handler):
             # The key is not available for the sample, so raise a KeyError
             raise KeyError("Metadata category %s does not exists for sample %s"
                            " in template %d" %
                            (key, self._id, self._md_template.id))
 
+        sql = """SELECT {0} FROM qiita.{1}
+                 WHERE sample_id=%s""".format(key, self._dynamic_table)
+
+        return conn_handler.execute_fetchone(sql, (self._id, ))[0]
+
     def add_setitem_queries(self, column, value, conn_handler, queue):
         """Adds the SQL queries needed to set a value to the provided queue
 
@@ -336,40 +288,15 @@ def add_setitem_queries(self, column, value, conn_handler, queue):
         QiitaDBColumnError
             If the column does not exist in the table
         """
-        # try dynamic tables
-        sql = """SELECT EXISTS (
-                    SELECT column_name
-                    FROM information_schema.columns
-                    WHERE table_name=%s
-                        AND table_schema='qiita'
-                        AND column_name=%s)"""
-        exists_dynamic = conn_handler.execute_fetchone(
-            sql, (self._dynamic_table, column))[0]
-        # try required_sample_info
-        sql = """SELECT EXISTS (
-                    SELECT column_name
-                    FROM information_schema.columns
-                    WHERE table_name=%s
-                        AND table_schema='qiita'
-                        AND column_name=%s)"""
-        exists_required = conn_handler.execute_fetchone(
-            sql, (self._table, column))[0]
-
-        if exists_dynamic:
-                sql = """UPDATE qiita.{0}
-                         SET {1}=%s
-                         WHERE sample_id=%s""".format(self._dynamic_table,
-                                                      column)
-        elif exists_required:
-            # here is not required the type check as the required fields have
-            # an explicit type check
-            sql = """UPDATE qiita.{0}
-                     SET {1}=%s
-                     WHERE sample_id=%s""".format(self._table, column)
-        else:
+        # Check if the column exist in the table
+        if column not in self._get_categories(conn_handler):
             raise QiitaDBColumnError("Column %s does not exist in %s" %
                                      (column, self._dynamic_table))
 
+        sql = """UPDATE qiita.{0}
+                 SET {1}=%s
+                 WHERE sample_id=%s""".format(self._dynamic_table, column)
+
         conn_handler.add_to_queue(queue, sql, (value, self._id))
 
     def __setitem__(self, column, value):

qiita_db/metadata_template/prep_template.py

@@ -15,7 +15,7 @@
                                  QiitaDBError, QiitaDBExecutionError)
 from qiita_db.sql_connection import SQLConnectionHandler
 from qiita_db.ontology import Ontology
-from qiita_db.util import (get_emp_status, convert_to_id,
+from qiita_db.util import (convert_to_id,
                            convert_from_id, get_mountpoint, infer_status)
 from .base_metadata_template import BaseSample, MetadataTemplate
 from .util import load_template_to_dataframe
@@ -31,7 +31,7 @@ class PrepSample(BaseSample):
     BaseSample
     Sample
     """
-    _table = "common_prep_info"
+    _table = "prep_template_sample"
     _table_prefix = "prep_"
     _column_table = "prep_columns"
     _id_column = "prep_template_id"
@@ -62,13 +62,11 @@ class PrepTemplate(MetadataTemplate):
     MetadataTemplate
     SampleTemplate
     """
-    _table = "common_prep_info"
+    _table = "prep_template_sample"
     _table_prefix = "prep_"
     _column_table = "prep_columns"
     _id_column = "prep_template_id"
     translate_cols_dict = {'emp_status_id': 'emp_status'}
-    id_cols_handlers = {'emp_status_id': get_emp_status()}
-    str_cols_handlers = {'emp_status_id': get_emp_status(key='emp_status_id')}
     _sample_cls = PrepSample
 
     @classmethod

qiita_db/metadata_template/sample_template.py

@@ -31,7 +31,7 @@ class Sample(BaseSample):
     BaseSample
     PrepSample
     """
-    _table = "required_sample_info"
+    _table = "study_sample"
     _table_prefix = "sample_"
     _column_table = "study_sample_columns"
     _id_column = "study_id"
@@ -62,17 +62,12 @@ class SampleTemplate(MetadataTemplate):
     MetadataTemplate
     PrepTemplate
     """
-    _table = "required_sample_info"
+    _table = "study_sample"
     _table_prefix = "sample_"
     _column_table = "study_sample_columns"
     _id_column = "study_id"
     translate_cols_dict = {
         'required_sample_info_status_id': 'required_sample_info_status'}
-    id_cols_handlers = {
-        'required_sample_info_status_id': get_required_sample_info_status()}
-    str_cols_handlers = {
-        'required_sample_info_status_id': get_required_sample_info_status(
-            key='required_sample_info_status_id')}
     _sample_cls = Sample
 
     @staticmethod

qiita_db/metadata_template/test/test_prep_template.py

@@ -71,9 +71,9 @@ def test_add_setitem_queries_required(self):
             'center_name', 'FOO', conn_handler, queue)
 
         obs = conn_handler.queues[queue]
-        sql = """UPDATE qiita.common_prep_info
-                     SET center_name=%s
-                     WHERE sample_id=%s"""
+        sql = """UPDATE qiita.prep_1
+                 SET center_name=%s
+                 WHERE sample_id=%s"""
         exp = [(sql, ('FOO', '1.SKB8.640193'))]
         self.assertEqual(obs, exp)
 
@@ -87,8 +87,8 @@ def test_add_setitem_queries_dynamic(self):
 
         obs = conn_handler.queues[queue]
         sql = """UPDATE qiita.prep_1
-                         SET barcodesequence=%s
-                         WHERE sample_id=%s"""
+                 SET barcodesequence=%s
+                 WHERE sample_id=%s"""
         exp = [(sql, ('AAAAAAAAAAAA', '1.SKB8.640193'))]
         self.assertEqual(obs, exp)
 
@@ -262,9 +262,6 @@ def test_setitem(self):
         with self.assertRaises(QiitaDBColumnError):
             self.tester['column that does not exist'] = 0.3
 
-        with self.assertRaises(ValueError):
-            self.tester['emp_status_id'] = "Error!"
-
         self.assertEqual(self.tester['center_name'], 'ANL')
         self.tester['center_name'] = "FOO"
         self.assertEqual(self.tester['center_name'], "FOO")

qiita_db/metadata_template/test/test_sample_template.py

@@ -38,9 +38,9 @@ def setUp(self):
         self.sample_template = SampleTemplate(1)
         self.sample_id = '1.SKB8.640193'
         self.tester = Sample(self.sample_id, self.sample_template)
-        self.exp_categories = {'physical_location', 'has_physical_specimen',
-                               'has_extracted_data', 'sample_type',
-                               'required_sample_info_status',
+        self.exp_categories = {'physical_specimen_location',
+                               'physical_specimen_remaining',
+                               'dna_extracted', 'sample_type',
                                'collection_timestamp', 'host_subject_id',
                                'description', 'season_environment',
                                'assigned_from_geo', 'texture', 'taxon_id',
@@ -68,12 +68,12 @@ def test_add_setitem_queries_required(self):
         conn_handler.create_queue(queue)
 
         self.tester.add_setitem_queries(
-            'has_physical_specimen', True, conn_handler, queue)
+            'physical_specimen_remaining', True, conn_handler, queue)
 
         obs = conn_handler.queues[queue]
-        sql = """UPDATE qiita.required_sample_info
-                     SET has_physical_specimen=%s
-                     WHERE sample_id=%s"""
+        sql = """UPDATE qiita.sample_1
+                 SET physical_specimen_remaining=%s
+                 WHERE sample_id=%s"""
         exp = [(sql, (True, '1.SKB8.640193'))]
         self.assertEqual(obs, exp)
 
@@ -87,8 +87,8 @@ def test_add_setitem_queries_dynamic(self):
 
         obs = conn_handler.queues[queue]
         sql = """UPDATE qiita.sample_1
-                         SET tot_nitro=%s
-                         WHERE sample_id=%s"""
+                 SET tot_nitro=%s
+                 WHERE sample_id=%s"""
         exp = [(sql, ('1234.5', '1.SKB8.640193'))]
         self.assertEqual(obs, exp)
 
@@ -144,28 +144,22 @@ def test_get_categories(self):
 
     def test_len(self):
         """Len returns the correct number of categories"""
-        self.assertEqual(len(self.tester), 30)
+        self.assertEqual(len(self.tester), 29)
 
     def test_getitem_required(self):
         """Get item returns the correct metadata value from the required table
         """
-        self.assertEqual(self.tester['physical_location'], 'ANL')
+        self.assertEqual(self.tester['physical_specimen_location'], 'ANL')
         self.assertEqual(self.tester['collection_timestamp'],
                          datetime(2011, 11, 11, 13, 00, 00))
-        self.assertTrue(self.tester['has_physical_specimen'])
+        self.assertTrue(self.tester['dna_extracted'])
 
     def test_getitem_dynamic(self):
         """Get item returns the correct metadata value from the dynamic table
         """
         self.assertEqual(self.tester['SEASON_ENVIRONMENT'], 'winter')
         self.assertEqual(self.tester['depth'], 0.15)
 
-    def test_getitem_id_column(self):
-        """Get item returns the correct metadata value from the changed column
-        """
-        self.assertEqual(self.tester['required_sample_info_status'],
-                         'completed')
-
     def test_getitem_error(self):
         """Get item raises an error if category does not exists"""
         with self.assertRaises(KeyError):
@@ -196,7 +190,7 @@ def test_values(self):
         """values returns an iterator over the values"""
         obs = self.tester.values()
         self.assertTrue(isinstance(obs, Iterable))
-        exp = {'ANL', True, True, 'ENVO:soil', 'completed',
+        exp = {'ANL', True, True, 'ENVO:soil',
                datetime(2011, 11, 11, 13, 00, 00), '1001:M7',
                'Cannabis Soil Microbiome', 'winter', 'n',
                '64.6 sand, 17.6 silt, 17.8 clay', '1118232', 0.15, '3483',
@@ -211,9 +205,10 @@ def test_items(self):
         """items returns an iterator over the (key, value) tuples"""
         obs = self.tester.items()
         self.assertTrue(isinstance(obs, Iterable))
-        exp = {('physical_location', 'ANL'), ('has_physical_specimen', True),
-               ('has_extracted_data', True), ('sample_type', 'ENVO:soil'),
-               ('required_sample_info_status', 'completed'),
+        exp = {('physical_specimen_location', 'ANL'),
+               ('physical_specimen_remaining', True),
+               ('dna_extracted', True),
+               ('sample_type', 'ENVO:soil'),
                ('collection_timestamp', datetime(2011, 11, 11, 13, 00, 00)),
                ('host_subject_id', '1001:M7'),
                ('description', 'Cannabis Soil Microbiome'),