Merge pull request #1108 from biocore/cart-branch

Cart branch

Author: ElDeveloper

qiita_db/analysis.py

@@ -18,6 +18,7 @@
 # -----------------------------------------------------------------------------
 from __future__ import division
 from collections import defaultdict
+from itertools import product
 from os.path import join
 
 from future.utils import viewitems
@@ -62,6 +63,7 @@ class Analysis(QiitaStatusObject):
     share
     unshare
     build_files
+    summary_data
     """
 
     _table = "analysis"
@@ -99,7 +101,7 @@ def get_by_status(cls, status):
         return {x[0] for x in conn_handler.execute_fetchall(sql, (status,))}
 
     @classmethod
-    def create(cls, owner, name, description, parent=None):
+    def create(cls, owner, name, description, parent=None, from_default=False):
         """Creates a new analysis on the database
 
         Parameters
@@ -112,23 +114,53 @@ def create(cls, owner, name, description, parent=None):
             Description of the analysis
         parent : Analysis object, optional
             The analysis this one was forked from
+        from_default : bool, optional
+            If True, use the default analysis to populate selected samples.
+            Default False.
         """
+        queue = "create_analysis"
         conn_handler = SQLConnectionHandler()
+        conn_handler.create_queue(queue)
         # TODO after demo: if exists()
-
-        # insert analysis information into table with "in construction" status
-        sql = ("INSERT INTO qiita.{0} (email, name, description, "
-               "analysis_status_id) VALUES (%s, %s, %s, 1) "
-               "RETURNING analysis_id".format(cls._table))
-        a_id = conn_handler.execute_fetchone(
-            sql, (owner.id, name, description))[0]
+        # Needed since issue #292 exists
+        status_id = conn_handler.execute_fetchone(
+            "SELECT analysis_status_id from qiita.analysis_status WHERE "
+            "status = 'in_construction'")[0]
+        if from_default:
+            # insert analysis and move samples into that new analysis
+            dflt_id = owner.default_analysis
+            sql = """INSERT INTO qiita.{0}
+                    (email, name, description, analysis_status_id)
+                    VALUES (%s, %s, %s, %s)
+                    RETURNING analysis_id""".format(cls._table)
+            conn_handler.add_to_queue(queue, sql, (owner.id, name,
+                                                   description, status_id))
+            # MAGIC NUMBER 3: command selection step
+            # needed so we skip the sample selection step
+            sql = """INSERT INTO qiita.analysis_workflow
+                    (analysis_id, step) VALUES (%s, %s)
+                    RETURNING %s"""
+            conn_handler.add_to_queue(queue, sql, ['{0}', 3, '{0}'])
+            sql = """UPDATE qiita.analysis_sample
+                     SET analysis_id = %s
+                     WHERE analysis_id = %s RETURNING %s"""
+            conn_handler.add_to_queue(queue, sql, ['{0}', dflt_id, '{0}'])
+        else:
+            # insert analysis information into table as "in construction"
+            sql = """INSERT INTO qiita.{0}
+                  (email, name, description, analysis_status_id)
+                  VALUES (%s, %s, %s, %s)
+                  RETURNING analysis_id""".format(cls._table)
+            conn_handler.add_to_queue(
+                queue, sql, (owner.id, name, description, status_id))
 
         # add parent if necessary
         if parent:
             sql = ("INSERT INTO qiita.analysis_chain (parent_id, child_id) "
-                   "VALUES (%s, %s)")
-            conn_handler.execute(sql, (parent.id, a_id))
+                   "VALUES (%s, %s) RETURNING child_id")
+            conn_handler.add_to_queue(queue, sql, [parent.id, '{0}'])
 
+        a_id = conn_handler.execute_queue(queue)[0]
         return cls(a_id)
 
     # ---- Properties ----
@@ -470,6 +502,23 @@ def has_access(self, user):
         return self._id in Analysis.get_by_status('public') | \
             user.private_analyses | user.shared_analyses
 
+    def summary_data(self):
+        """Return number of studies, processed data, and samples selected
+
+        Returns
+        -------
+        dict
+            counts keyed to their relevant type
+        """
+        sql = """SELECT COUNT(DISTINCT study_id) as studies,
+                COUNT(DISTINCT processed_data_id) as processed_data,
+                COUNT(DISTINCT sample_id) as samples
+                FROM qiita.study_processed_data
+                JOIN qiita.analysis_sample USING (processed_data_id)
+                WHERE analysis_id = %s"""
+        conn_handler = SQLConnectionHandler()
+        return dict(conn_handler.execute_fetchone(sql, [self._id]))
+
     def share(self, user):
         """Share the analysis with another user
 
@@ -511,17 +560,26 @@ def add_samples(self, samples):
 
         Parameters
         ----------
-        samples : list of tuples of (int, str)
+        samples : dictionary of lists
             samples and the processed data id they come from in form
-            [(processed_data_id, sample_id), ...]
+            {processed_data_id: [sample1, sample2, ...], ...}
         """
         conn_handler = SQLConnectionHandler()
         self._lock_check(conn_handler)
-        sql = ("INSERT INTO qiita.analysis_sample "
-               "(analysis_id, processed_data_id, sample_id) VALUES "
-               "(%s, %s, %s)")
-        conn_handler.executemany(sql, [(self._id, s[0], s[1])
-                                       for s in samples])
+
+        for pid, samps in viewitems(samples):
+            # get previously selected samples  for pid and filter them out
+            sql = """SELECT sample_id FROM qiita.analysis_sample
+                WHERE processed_data_id = %s and analysis_id = %s"""
+            prev_selected = [x[0] for x in
+                             conn_handler.execute_fetchall(sql,
+                                                           (pid, self._id))]
+
+            select = set(samps).difference(prev_selected)
+            sql = ("INSERT INTO qiita.analysis_sample "
+                   "(analysis_id, processed_data_id, sample_id) VALUES "
+                   "({}, %s, %s)".format(self._id))
+            conn_handler.executemany(sql, [x for x in product([pid], select)])
 
     def remove_samples(self, proc_data=None, samples=None):
         """Removes samples from the analysis

qiita_db/data.py

@@ -1425,12 +1425,53 @@ def data_type(self, ret_id=False):
         return data_type[0]
 
     @property
-    def processed_date(self):
-        """Return the processed date"""
+    def processing_info(self):
+        """Return the processing item and settings used to create the data
+
+        Returns
+        -------
+        dict
+            Parameter settings keyed to the parameter, along with date and
+            algorithm used
+        """
+        # Get processed date and the info for the dynamic table
         conn_handler = SQLConnectionHandler()
-        return conn_handler.execute_fetchone(
-            "SELECT processed_date FROM qiita.{0} WHERE "
-            "processed_data_id=%s".format(self._table), (self.id,))[0]
+        sql = """SELECT processed_date, processed_params_table,
+            processed_params_id FROM qiita.{0}
+            WHERE processed_data_id=%s""".format(self._table)
+        static_info = conn_handler.execute_fetchone(sql, (self.id,))
+
+        # Get the info from the dynamic table, including reference used
+        sql = """SELECT * from qiita.{0}
+            JOIN qiita.reference USING (reference_id)
+            WHERE processed_params_id = {1}
+            """.format(static_info['processed_params_table'],
+                       static_info['processed_params_id'])
+        dynamic_info = dict(conn_handler.execute_fetchone(sql))
+
+        # replace reference filepath_ids with full filepaths
+        # figure out what columns have filepaths and what don't
+        ref_fp_cols = {'sequence_filepath', 'taxonomy_filepath',
+                       'tree_filepath'}
+        fp_ids = [str(dynamic_info[col]) for col in ref_fp_cols
+                  if dynamic_info[col] is not None]
+        # Get the filepaths and create dict of fpid to filepath
+        sql = ("SELECT filepath_id, filepath FROM qiita.filepath WHERE "
+               "filepath_id IN ({})").format(','.join(fp_ids))
+        lookup = {fp[0]: fp[1] for fp in conn_handler.execute_fetchall(sql)}
+        # Loop through and replace ids
+        for key in ref_fp_cols:
+            if dynamic_info[key] is not None:
+                dynamic_info[key] = lookup[dynamic_info[key]]
+
+        # add missing info to the dictionary and remove id column info
+        dynamic_info['processed_date'] = static_info['processed_date']
+        dynamic_info['algorithm'] = static_info[
+            'processed_params_table'].split('_')[-1]
+        del dynamic_info['processed_params_id']
+        del dynamic_info['reference_id']
+
+        return dynamic_info
 
     @property
     def samples(self):

qiita_db/support_files/patches/21.sql

@@ -0,0 +1,14 @@
+-- March 28, 2015
+-- Add default analyses for all existing users
+DO $do$
+DECLARE 
+	eml varchar;
+	aid bigint;
+BEGIN
+FOR eml IN
+	SELECT email FROM qiita.qiita_user
+LOOP
+	INSERT INTO qiita.analysis (email, name, description, dflt, analysis_status_id) VALUES (eml, eml || '-dflt', 'dflt', true, 1) RETURNING analysis_id INTO aid;
+	INSERT INTO qiita.analysis_workflow (analysis_id, step) VALUES (aid, 2);
+END LOOP;
+END $do$;

qiita_db/support_files/patches/22.sql

@@ -0,0 +1,14 @@
+-- April 16, 2015
+-- Add primary key to analysis_sample table, first deleting the duplicates
+-- http://stackoverflow.com/a/9862688
+DO $do$
+BEGIN
+    CREATE TEMP TABLE temp_table
+    ON COMMIT drop AS
+        SELECT analysis_id, processed_data_id, sample_id
+        FROM qiita.analysis_sample GROUP BY analysis_id, processed_data_id, sample_id;
+    DELETE FROM qiita.analysis_sample;
+    INSERT INTO qiita.analysis_sample (analysis_id, processed_data_id, sample_id) SELECT analysis_id, processed_data_id, sample_id FROM temp_table;
+
+    ALTER TABLE qiita.analysis_sample ADD CONSTRAINT pk_analysis_sample PRIMARY KEY ( analysis_id, processed_data_id, sample_id );
+END $do$

qiita_db/support_files/populate_test_db.sql

@@ -420,3 +420,9 @@ INSERT INTO qiita.collection_job (collection_id, job_id) VALUES (1, 1);
 
 --share collection with shared user
 INSERT INTO qiita.collection_users (email, collection_id) VALUES ('shared@foo.bar', 1);
+
+--add default analysis for users
+INSERT INTO qiita.analysis (email, name, description, dflt, analysis_status_id) VALUES ('test@foo.bar', 'test@foo.bar-dflt', 'dflt', true, 1), ('admin@foo.bar', 'admin@foo.bar-dflt', 'dflt', true, 1), ('shared@foo.bar', 'shared@foo.bar-dflt', 'dflt', true, 1), ('demo@microbio.me', 'demo@microbio.me-dflt', 'dflt', true, 1);
+
+-- Attach samples to analysis
+INSERT INTO qiita.analysis_sample (analysis_id, processed_data_id, sample_id) VALUES (3,1,'1.SKD8.640184'), (3,1,'1.SKB7.640196'), (3,1,'1.SKM9.640192'), (3,1,'1.SKM4.640180')

qiita_db/support_files/qiita-db.dbs

@@ -127,6 +127,11 @@
 			<index name="idx_analysis_sample_1" unique="NORMAL" >
 				<column name="sample_id" />
 			</index>
+			<index name="pk_analysis_sample" unique="PRIMARY_KEY" >
+				<column name="analysis_id" />
+				<column name="processed_data_id" />
+				<column name="sample_id" />
+			</index>
 			<fk name="fk_analysis_sample_analysis" to_schema="qiita" to_table="analysis" >
 				<fk_column name="analysis_id" pk="analysis_id" />
 			</fk>
@@ -1611,7 +1616,6 @@ Controlled Vocabulary]]></comment>
 		<entity schema="qiita" name="data_directory" color="b2cdf7" x="840" y="585" />
 		<entity schema="qiita" name="term" color="d0def5" x="810" y="1650" />
 		<entity schema="qiita" name="common_prep_info" color="d0def5" x="1050" y="165" />
-		<entity schema="qiita" name="analysis_sample" color="d0def5" x="45" y="1170" />
 		<entity schema="qiita" name="environmental_package" color="b2cdf7" x="2250" y="150" />
 		<entity schema="qiita" name="study_environmental_package" color="b2cdf7" x="2250" y="45" />
 		<entity schema="qiita" name="timeseries_type" color="c0d4f3" x="1680" y="615" />
@@ -1639,6 +1643,7 @@ Controlled Vocabulary]]></comment>
 		<entity schema="qiita" name="investigation" color="c0d4f3" x="2100" y="255" />
 		<entity schema="qiita" name="processed_data_status" color="c0d4f3" x="1500" y="1050" />
 		<entity schema="qiita" name="portal_type" color="c0d4f3" x="1995" y="660" />
+		<entity schema="qiita" name="analysis_sample" color="d0def5" x="45" y="1170" />
 		<group name="Group_analyses" color="c4e0f9" >
 			<comment>analysis tables</comment>
 			<entity schema="qiita" name="analysis" />