Issue 855

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+sudo: false
 env:
   global:
     - PYTHON_VERSION=2.7

qiita_db/data.py

@@ -85,10 +85,11 @@
 from .base import QiitaObject
 from .logger import LogEntry
 from .sql_connection import SQLConnectionHandler
-from .exceptions import QiitaDBError, QiitaDBUnknownIDError
+from .exceptions import QiitaDBError, QiitaDBUnknownIDError, QiitaDBStatusError
 from .util import (exists_dynamic_table, insert_filepaths, convert_to_id,
                    convert_from_id, purge_filepaths, get_filepath_id,
-                   get_mountpoint, move_filepaths_to_upload_folder)
+                   get_mountpoint, move_filepaths_to_upload_folder,
+                   infer_status)
 
 
 class BaseData(QiitaObject):
@@ -615,6 +616,58 @@ def remove_filepath(self, fp):
         # Delete the files, if they are not used anywhere
         purge_filepaths(conn_handler)
 
+    def status(self, study):
+        """The status of the raw data within the given study
+
+        Parameters
+        ----------
+        study : Study
+            The study that is looking to the raw data status
+
+        Returns
+        -------
+        str
+            The status of the raw data
+
+        Raises
+        ------
+        QiitaDBStatusError
+            If the raw data does not belong to the passed study
+
+        Notes
+        -----
+        Given that a raw data can be shared by multiple studies, we need to
+        know under which context (study) we want to check the raw data status.
+        The rationale is that a raw data object can contain data from multiple
+        studies, so the raw data can have multiple status at the same time.
+        We then check the processed data generated to infer the status of the
+        raw data.
+        """
+        if self._id not in study.raw_data():
+            raise QiitaDBStatusError(
+                "The study %s does not have access to the raw data %s"
+                % (study.id, self.id))
+
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT processed_data_status
+                FROM qiita.processed_data_status pds
+                  JOIN qiita.processed_data pd
+                    USING (processed_data_status_id)
+                  JOIN qiita.preprocessed_processed_data ppd_pd
+                    USING (processed_data_id)
+                  JOIN qiita.prep_template_preprocessed_data pt_ppd
+                    USING (preprocessed_data_id)
+                  JOIN qiita.prep_template pt
+                    USING (prep_template_id)
+                  JOIN qiita.raw_data rd
+                    USING (raw_data_id)
+                  JOIN qiita.study_processed_data spd
+                    USING (processed_data_id)
+                WHERE pt.raw_data_id=%s AND spd.study_id=%s"""
+        pd_statuses = conn_handler.execute_fetchall(sql, (self._id, study.id))
+
+        return infer_status(pd_statuses)
+
 
 class PreprocessedData(BaseData):
     r"""Object for dealing with preprocessed data
@@ -995,6 +1048,34 @@ def processing_status(self, state):
             "UPDATE qiita.{0} SET processing_status=%s WHERE "
             "preprocessed_data_id=%s".format(self._table), (state, self.id))
 
+    @property
+    def status(self):
+        """The status of the preprocessed data
+
+        Returns
+        -------
+        str
+            The status of the preprocessed_data
+
+        Notes
+        -----
+        The status of a preprocessed data is inferred by the status of the
+        processed data generated from this preprocessed data. If no processed
+        data has been generated with this preprocessed data; then the status
+        is 'sandbox'.
+        """
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT processed_data_status
+                FROM qiita.processed_data_status pds
+                  JOIN qiita.processed_data pd
+                    USING (processed_data_status_id)
+                  JOIN qiita.preprocessed_processed_data ppd_pd
+                    USING (processed_data_id)
+                WHERE ppd_pd.preprocessed_data_id=%s"""
+        pd_statuses = conn_handler.execute_fetchall(sql, (self._id,))
+
+        return infer_status(pd_statuses)
+
 
 class ProcessedData(BaseData):
     r"""Object for dealing with processed data
@@ -1020,6 +1101,61 @@ class ProcessedData(BaseData):
     _study_processed_table = "study_processed_data"
     _preprocessed_processed_table = "preprocessed_processed_data"
 
+    @classmethod
+    def get_by_status(cls, status):
+        """Returns id for all ProcessedData with given status
+
+        Parameters
+        ----------
+        status : str
+            Status to search for
+
+        Returns
+        -------
+        list of int
+            All the processed data ids that match the given status
+        """
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT processed_data_id FROM qiita.processed_data pd
+                JOIN qiita.processed_data_status pds
+                    USING (processed_data_status_id)
+                WHERE pds.processed_data_status=%s"""
+        result = conn_handler.execute_fetchall(sql, (status,))
+        if result:
+            pds = set(x[0] for x in result)
+        else:
+            pds = set()
+
+        return pds
+
+    @classmethod
+    def get_by_status_grouped_by_study(cls, status):
+        """Returns id for all ProcessedData with given status grouped by study
+
+        Parameters
+        ----------
+        status : str
+            Status to search for
+
+        Returns
+        -------
+        dict of list of int
+            A dictionary keyed by study id in which the values are the
+            processed data ids that belong to that study and match the given
+            status
+        """
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT spd.study_id,
+            array_agg(pd.processed_data_id ORDER BY pd.processed_data_id)
+            FROM qiita.processed_data pd
+                JOIN qiita.processed_data_status pds
+                    USING (processed_data_status_id)
+                JOIN qiita.study_processed_data spd
+                    USING (processed_data_id)
+            WHERE pds.processed_data_status = %s
+            GROUP BY spd.study_id;"""
+        return dict(conn_handler.execute_fetchall(sql, (status,)))
+
     @classmethod
     def create(cls, processed_params_table, processed_params_id, filepaths,
                preprocessed_data=None, study=None, processed_date=None,
@@ -1173,3 +1309,40 @@ def processed_date(self):
         return conn_handler.execute_fetchone(
             "SELECT processed_date FROM qiita.{0} WHERE "
             "processed_data_id=%s".format(self._table), (self.id,))[0]
+
+    @property
+    def status(self):
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT pds.processed_data_status
+                FROM qiita.processed_data_status pds
+                  JOIN qiita.processed_data pd
+                    USING (processed_data_status_id)
+                WHERE pd.processed_data_id=%s"""
+        return conn_handler.execute_fetchone(sql, (self._id,))[0]
+
+    @status.setter
+    def status(self, status):
+        """Set the status value
+
+        Parameters
+        ----------
+        status : str
+            The new status
+
+        Raises
+        ------
+        QiitaDBStatusError
+            If the processed data status is public
+        """
+        if self.status == 'public':
+            raise QiitaDBStatusError(
+                "Illegal operation on public processed data")
+
+        conn_handler = SQLConnectionHandler()
+
+        status_id = convert_to_id(status, 'processed_data_status',
+                                  conn_handler=conn_handler)
+
+        sql = """UPDATE qiita.{0} SET processed_data_status_id = %s
+                 WHERE processed_data_id=%s""".format(self._table)
+        conn_handler.execute(sql, (status_id, self._id))

qiita_db/meta_util.py

@@ -81,9 +81,8 @@ def get_accessible_filepath_ids(user):
         return set(f[0] for f in fpids)
 
     # First, the studies
-    # There are public, private, and shared studies
-    study_ids = Study.get_by_status('public') | user.user_studies | \
-        user.shared_studies
+    # There are private and shared studies
+    study_ids = user.user_studies | user.shared_studies
 
     filepath_ids = set()
     for study_id in study_ids:
@@ -107,7 +106,6 @@ def get_accessible_filepath_ids(user):
         for rdid in study.raw_data():
             for pt_id in RawData(rdid).prep_templates:
                 # related to https://github.com/biocore/qiita/issues/596
-                # and https://github.com/biocore/qiita/issues/554
                 if PrepTemplate.exists(pt_id):
                     for _id, _ in PrepTemplate(pt_id).get_filepaths():
                         prep_fp_ids.append(_id)
@@ -118,6 +116,40 @@ def get_accessible_filepath_ids(user):
                              in SampleTemplate(study_id).get_filepaths()]
             filepath_ids.update(sample_fp_ids)
 
+    # Next, the public processed data
+    processed_data_ids = ProcessedData.get_by_status('public')
+    for pd_id in processed_data_ids:
+        processed_data = ProcessedData(pd_id)
+
+        # Add the filepaths of the processed data
+        pd_fps = (fpid for fpid, _, _ in processed_data.get_filepaths())
+        filepath_ids.update(pd_fps)
+
+        # Each processed data has a preprocessed data
+        ppd = PreprocessedData(processed_data.preprocessed_data)
+        ppd_fps = (fpid for fpid, _, _ in ppd.get_filepaths())
+        filepath_ids.update(ppd_fps)
+
+        # Each preprocessed data has a prep template
+        pt_id = ppd.prep_template
+        # related to https://github.com/biocore/qiita/issues/596
+        if PrepTemplate.exists(pt_id):
+            pt = PrepTemplate(pt_id)
+            pt_fps = (fpid for fpid, _ in pt.get_filepaths())
+            filepath_ids.update(pt_fps)
+
+            # Each prep template has a raw data
+            rd = RawData(pt.raw_data)
+            rd_fps = (fpid for fpid, _, _ in rd.get_filepaths())
+            filepath_ids.update(rd_fps)
+
+        # And each processed data has a study, which has a sample template
+        st_id = processed_data.study
+        if SampleTemplate.exists(st_id):
+            sample_fp_ids = (_id for _id, _
+                             in SampleTemplate(st_id).get_filepaths())
+            filepath_ids.update(sample_fp_ids)
+
     # Next, analyses
     # Same as before, there are public, private, and shared
     analysis_ids = Analysis.get_by_status('public') | user.private_analyses | \

qiita_db/metadata_template.py

@@ -62,7 +62,7 @@
 from .util import (exists_table, get_table_cols, get_emp_status,
                    get_required_sample_info_status, convert_to_id,
                    convert_from_id, get_mountpoint, insert_filepaths,
-                   scrub_data)
+                   scrub_data, infer_status)
 from .study import Study
 from .data import RawData
 from .logger import LogEntry
@@ -2206,6 +2206,36 @@ def create_qiime_mapping_file(self, prep_template_fp):
 
         return filepath
 
+    @property
+    def status(self):
+        """The status of the prep template
+
+        Returns
+        -------
+        str
+            The status of the prep template
+
+        Notes
+        -----
+        The status of a prep template is inferred by the status of the
+        processed data generated from this prep template. If no processed
+        data has been generated with this prep template; then the status
+        is 'sandbox'.
+        """
+        conn_handler = SQLConnectionHandler()
+        sql = """SELECT processed_data_status
+                FROM qiita.processed_data_status pds
+                  JOIN qiita.processed_data pd
+                    USING (processed_data_status_id)
+                  JOIN qiita.preprocessed_processed_data ppd_pd
+                    USING (processed_data_id)
+                  JOIN qiita.prep_template_preprocessed_data pt_ppd
+                    USING (preprocessed_data_id)
+                WHERE pt_ppd.prep_template_id=%s"""
+        pd_statuses = conn_handler.execute_fetchall(sql, (self._id,))
+
+        return infer_status(pd_statuses)
+
 
 def load_template_to_dataframe(fn, strip_whitespace=True):
     """Load a sample or a prep template into a data frame