merge upstream/master

Author: squirrelo

qiita_db/data.py

@@ -85,7 +85,7 @@
 from .base import QiitaObject
 from .logger import LogEntry
 from .sql_connection import SQLConnectionHandler
-from .exceptions import QiitaDBError
+from .exceptions import QiitaDBError, QiitaDBUnknownIDError
 from .util import (exists_dynamic_table, insert_filepaths, convert_to_id,
                    convert_from_id, purge_filepaths, get_filepath_id,
                    get_mountpoint, move_filepaths_to_upload_folder)
@@ -243,6 +243,27 @@ def _set_link_filepaths_status(self, status):
             "WHERE {1} = %s".format(self._table, self._data_filepath_column),
             (status, self._id))
 
+    @classmethod
+    def exists(cls, object_id):
+        r"""Checks if the given object_id exists
+
+        Parameters
+        ----------
+        id : str
+            The id of the object we are searching for
+
+        Returns
+        -------
+        bool
+            True if exists, false otherwise.
+        """
+        conn_handler = SQLConnectionHandler()
+
+        return conn_handler.execute_fetchone(
+            "SELECT EXISTS(SELECT * FROM qiita.{0} WHERE "
+            "{1}=%s)".format(cls._table, cls._data_filepath_column),
+            (object_id, ))[0]
+
 
 class RawData(BaseData):
     r"""Object for dealing with raw data
@@ -308,6 +329,81 @@ def create(cls, filetype, studies, filepaths=None):
 
         return rd
 
+    @classmethod
+    def delete(cls, raw_data_id, study_id):
+        """Removes the raw data with id raw_data_id
+
+        Parameters
+        ----------
+        raw_data_id : int
+            The raw data id
+        study_id : int
+            The study id
+
+        Raises
+        ------
+        QiitaDBUnknownIDError
+            If the raw data id doesn't exist
+        QiitaDBError
+            If the raw data is not linked to that study_id
+            If the raw data has prep templates associated
+        """
+        conn_handler = SQLConnectionHandler()
+
+        # check if the raw data exist
+        if not cls.exists(raw_data_id):
+            raise QiitaDBUnknownIDError(raw_data_id, "raw data")
+
+        study_raw_data_exists = conn_handler.execute_fetchone(
+            "SELECT EXISTS(SELECT * FROM qiita.study_raw_data WHERE "
+            "study_id = {0} AND raw_data_id = {1})".format(study_id,
+                                                           raw_data_id))[0]
+        if not study_raw_data_exists:
+            raise QiitaDBError(
+                "Raw data %d is not linked to study %d or the study "
+                "doesn't exist" % (raw_data_id, study_id))
+
+        # check if there are any prep templates for this study
+        prep_template_exists = conn_handler.execute_fetchone(
+            """
+            SELECT EXISTS(
+                SELECT * FROM qiita.prep_template AS pt
+                    LEFT JOIN qiita.common_prep_info AS cpi ON
+                    (pt.prep_template_id=cpi.prep_template_id)
+                    LEFT JOIN qiita.required_sample_info AS rsi ON
+                    (cpi.sample_id=rsi.sample_id)
+                WHERE raw_data_id = {0} and study_id = {1}
+            )
+            """.format(raw_data_id, study_id))[0]
+        if prep_template_exists:
+            raise QiitaDBError(
+                "Raw data %d has prep template(s) associated so it can't be "
+                "erased." % raw_data_id)
+
+        # check how many raw data are left, if last one, check that there
+        # are no linked files
+        raw_data_count = conn_handler.execute_fetchone(
+            "SELECT COUNT(*) FROM qiita.study_raw_data WHERE "
+            "raw_data_id = {0}".format(raw_data_id))[0]
+        if raw_data_count == 1 and RawData(raw_data_id).get_filepath_ids():
+            raise QiitaDBError(
+                "Raw data (%d) can't be remove because it has linked files. "
+                "To remove it, first unlink files." % raw_data_id)
+
+        # delete
+        conn_handler.execute("DELETE FROM qiita.study_raw_data WHERE "
+                             "raw_data_id = {0} AND "
+                             "study_id = {1}".format(raw_data_id, study_id))
+
+        # delete the connecting tables if there is no other study linked to
+        # the raw data
+        study_raw_data_count = conn_handler.execute_fetchone(
+            "SELECT COUNT(*) FROM qiita.study_raw_data WHERE "
+            "raw_data_id = {0}".format(raw_data_id))[0]
+        if study_raw_data_count == 0:
+            conn_handler.execute("DELETE FROM qiita.raw_data WHERE "
+                                 "raw_data_id = {0}".format(raw_data_id))
+
     @property
     def studies(self):
         r"""The IDs of the studies to which this raw data belongs

qiita_db/metadata_template.py

@@ -1858,7 +1858,7 @@ def create_qiime_mapping_file(self, prep_template_fp):
                                          str(pt_sample_names-st_sample_names)))
 
         mapping = pt.join(st, lsuffix="_prep")
-        mapping.rename(columns=rename_cols, inplace=True, index=str.lower)
+        mapping.rename(columns=rename_cols, inplace=True)
 
         # Gets the orginal mapping columns and readjust the order to comply
         # with QIIME requirements

qiita_db/support_files/patches/14.sql

@@ -1,4 +1,4 @@
-
 --Feb 11, 2015
 --Placeholder for python to run
-select * from settings;
+-- we don't really need an SQL patch, but having nothing here breaks the build
+select 42;

qiita_db/support_files/patches/15.sql

@@ -0,0 +1,4 @@
+--Feb 11, 2015
+--Placeholder for python to run
+-- we don't really need an SQL patch, but having nothing here breaks the build
+select 42;

qiita_db/support_files/patches/python_patches/14.py

@@ -1,26 +1,74 @@
-# Feb 11, 2015
-# This changes all analysis files to be relative path instead of absolute
+# Feberuary 7, 2015
+# This patch recreates all the QIIME mapping files to avoid lower/upper case
+# problems. See https://github.com/biocore/qiita/issues/799
+#
+# heavily based on 7.py
 
-from os.path import basename, dirname
+from os.path import basename
+
+from skbio.util import flatten
 
-from qiita_db.util import get_mountpoint
 from qiita_db.sql_connection import SQLConnectionHandler
+from qiita_db.metadata_template import PrepTemplate
 
 conn_handler = SQLConnectionHandler()
 
-filepaths = conn_handler.execute_fetchall(
-    'SELECT f.* from qiita.filepath f JOIN qiita.analysis_filepath afp ON '
-    'f.filepath_id = afp.filepath_id')
-
-# retrieve relative filepaths as dictionary for matching
-mountpoints = {m[1].rstrip('/\\'): m[0] for m in get_mountpoint(
-    'analysis', conn_handler=conn_handler, retrieve_all=True)}
-
-for filepath in filepaths:
-    filename = basename(filepath['filepath'])
-    # find the ID of the analysis filepath used
-    mp_id = mountpoints[dirname(filepath['filepath']).rstrip('/\\')]
-    conn_handler.execute(
-        'UPDATE qiita.filepath SET filepath = %s, data_directory_id = %s WHERE'
-        ' filepath_id = %s',
-        [filename, mp_id, filepath['filepath_id']])
+sql = "SELECT prep_template_id FROM qiita.prep_template"
+all_ids = conn_handler.execute_fetchall(sql)
+
+q_name = 'unlink-bad-mapping-files'
+conn_handler.create_queue(q_name)
+
+# remove all the bad mapping files
+for prep_template_id in all_ids:
+
+    prep_template_id = prep_template_id[0]
+    pt = PrepTemplate(prep_template_id)
+    fps = pt.get_filepaths()
+
+    # get the QIIME mapping file, note that the way to figure out what is and
+    # what's not a qiime mapping file is to check for the existance of the
+    # word qiime in the basename of the file path, hacky but that's the way
+    # it is being done in qiita_pet/uimodules/raw_data_tab.py
+    mapping_files = [f for f in fps if '_qiime_' in basename(f[1])]
+
+    table = 'prep_template_filepath'
+    column = 'prep_template_id'
+
+    # unlink all the qiime mapping files for this prep template object
+    for mf in mapping_files:
+
+        # (1) get the ids that we are going to delete.
+        # because of the FK restriction, we cannot just delete the ids
+        ids = conn_handler.execute_fetchall(
+            'SELECT filepath_id FROM qiita.{0} WHERE '
+            '{1}=%s and filepath_id=%s'.format(table, column), (pt.id, mf[0]))
+        ids = flatten(ids)
+
+        # (2) delete the entries from the prep_template_filepath table
+        conn_handler.add_to_queue(
+            q_name, "DELETE FROM qiita.{0} "
+            "WHERE {1}=%s and filepath_id=%s;".format(table, column),
+            (pt.id, mf[0]))
+
+        # (3) delete the entries from the filepath table
+        conn_handler.add_to_queue(
+            q_name,
+            "DELETE FROM qiita.filepath WHERE "
+            "filepath_id IN ({0});".format(', '.join(map(str, ids))))
+
+try:
+    conn_handler.execute_queue(q_name)
+except Exception as e:
+    raise
+
+# create correct versions of the mapping files
+for prep_template_id in all_ids:
+
+    prep_template_id = prep_template_id[0]
+    pt = PrepTemplate(prep_template_id)
+
+    # we can guarantee that all the filepaths will be prep templates so
+    # we can just generate the qiime mapping files
+    for _, fpt in pt.get_filepaths():
+        pt.create_qiime_mapping_file(fpt)

qiita_db/support_files/patches/python_patches/15.py

@@ -0,0 +1,26 @@
+# Feb 11, 2015
+# This changes all analysis files to be relative path instead of absolute
+
+from os.path import basename, dirname
+
+from qiita_db.util import get_mountpoint
+from qiita_db.sql_connection import SQLConnectionHandler
+
+conn_handler = SQLConnectionHandler()
+
+filepaths = conn_handler.execute_fetchall(
+    'SELECT f.* from qiita.filepath f JOIN qiita.analysis_filepath afp ON '
+    'f.filepath_id = afp.filepath_id')
+
+# retrieve relative filepaths as dictionary for matching
+mountpoints = {m[1].rstrip('/\\'): m[0] for m in get_mountpoint(
+    'analysis', conn_handler=conn_handler, retrieve_all=True)}
+
+for filepath in filepaths:
+    filename = basename(filepath['filepath'])
+    # find the ID of the analysis filepath used
+    mp_id = mountpoints[dirname(filepath['filepath']).rstrip('/\\')]
+    conn_handler.execute(
+        'UPDATE qiita.filepath SET filepath = %s, data_directory_id = %s WHERE'
+        ' filepath_id = %s',
+        [filename, mp_id, filepath['filepath_id']])