Workflows GUI (#3078)

* fix #3076

* addressing @ElDeveloper comments

* adding workflows GUI

* improve HTML

* db changes and improvements

* comment out new db changes

* rm default_workflows from PluginHandler.get

* Apply suggestions from code review [skip ci]

Co-authored-by: Yoshiki Vázquez Baeza <yoshiki@ucsd.edu>

Co-authored-by: Yoshiki Vázquez Baeza <yoshiki@ucsd.edu>

Author: antgonza

.github/workflows/qiita-ci.yml

@@ -174,7 +174,6 @@ jobs:
           COVER_PACKAGE: ${{ matrix.cover_package }}
         run: |
           conda activate qiita
-          bash -c 'source /home/runner/.profile; conda activate qtp-biom; start_biom --help'
           export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
           export REDBIOM_HOST="http://localhost:7379"

qiita_db/handlers/plugin.py

@@ -83,8 +83,6 @@ def get(self, name, version):
                 'commands': [c.name for c in plugin.commands],
                 'publications': [{'DOI': doi, 'PubMed': pubmed}
                                  for doi, pubmed in plugin.publications],
-                'default_workflows': [w.name
-                                      for w in plugin.default_workflows],
                 'type': plugin.type,
                 'active': plugin.active}
         self.write(response)

qiita_db/handlers/tests/test_plugin.py

@@ -61,9 +61,6 @@ def test_get(self):
                          'Single Rarefaction'],
             'publications': [{'DOI': '10.1038/nmeth.f.303',
                               'PubMed': '20383131'}],
-            'default_workflows': ['FASTQ upstream workflow',
-                                  'FASTA upstream workflow',
-                                  'Per sample FASTQ upstream workflow'],
             'type': 'artifact transformation',
             'active': False}
         self.assertEqual(loads(obs.body), exp)

qiita_db/processing_job.py

@@ -1912,10 +1912,13 @@ def from_default_workflow(cls, user, dflt_wf, req_params, name=None,
             # [0] in_degrees returns a tuple, where [0] is the element we want
             all_nodes = {}
             roots = {}
+
             for node, position in in_degrees:
+                dp = node.default_parameter
+                cmd = dp.command
                 if position == 0:
-                    roots[node] = (node.command, node.parameters)
-                all_nodes[node] = (node.command, node.parameters)
+                    roots[node] = (cmd, dp)
+                all_nodes[node] = (cmd, dp)
 
             # Check that we have all the required parameters
             root_cmds = set(c for c, _ in roots.values())

qiita_db/software.py

@@ -1232,24 +1232,6 @@ def start_script(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
-    @property
-    def default_workflows(self):
-        """Returns the default workflows attached to the current software
-
-        Returns
-        -------
-        generator of qiita_db.software.DefaultWorkflow
-            The defaultworkflows attached to the software
-        """
-        with qdb.sql_connection.TRN:
-            sql = """SELECT default_workflow_id
-                     FROM qiita.default_workflow
-                     WHERE software_id = %s
-                     ORDER BY default_workflow_id"""
-            qdb.sql_connection.TRN.add(sql, [self.id])
-            for wf_id in qdb.sql_connection.TRN.execute_fetchflatten():
-                yield DefaultWorkflow(wf_id)
-
     @property
     def type(self):
         """Returns the type of the software
@@ -1764,23 +1746,7 @@ class DefaultWorkflowNode(qdb.base.QiitaObject):
     _table = "default_workflow_node"
 
     @property
-    def command(self):
-        """The command to execute in this node
-
-        Returns
-        -------
-        qiita_db.software.Command
-        """
-        with qdb.sql_connection.TRN:
-            sql = """SELECT command_id
-                     FROM qiita.default_workflow_node
-                     WHERE default_workflow_node_id = %s"""
-            qdb.sql_connection.TRN.add(sql, [self.id])
-            cmd_id = qdb.sql_connection.TRN.execute_fetchlast()
-            return qdb.software.Command(cmd_id)
-
-    @property
-    def parameters(self):
+    def default_parameter(self):
         """The default parameter set to use in this node
 
         Returns
@@ -1839,6 +1805,58 @@ class DefaultWorkflow(qdb.base.QiitaObject):
     """
     _table = "default_workflow"
 
+    @classmethod
+    def iter(cls, active=True):
+        """Iterates over all active DefaultWorkflow
+
+        Parameters
+        ----------
+        active : bool, optional
+            If True will only return active software
+
+        Returns
+        -------
+        list of qiita_db.software.DefaultWorkflow
+            The DefaultWorkflow objects
+        """
+        sql = """SELECT default_workflow_id
+                 FROM qiita.default_workflow {0}
+                 ORDER BY default_workflow_id""".format(
+                    'WHERE active = True' if active else '')
+        with qdb.sql_connection.TRN:
+            qdb.sql_connection.TRN.add(sql)
+            for s_id in qdb.sql_connection.TRN.execute_fetchflatten():
+                yield cls(s_id)
+
+    @property
+    def active(self):
+        """Retrieves active status of the default workflow
+
+        Retruns
+        ----------
+        active : bool
+            active value
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT active
+                     FROM qiita.default_workflow
+                     WHERE default_workflow_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
+    @active.setter
+    def active(self, active):
+        """Changes active status of the default workflow
+
+        Parameters
+        ----------
+        active : bool
+            New active value
+        """
+        sql = """UPDATE qiita.default_workflow SET active = %s
+                 WHERE default_workflow_id = %s"""
+        qdb.sql_connection.perform_as_transaction(sql, [active, self._id])
+
     @property
     def name(self):
         with qdb.sql_connection.TRN:
@@ -1848,6 +1866,23 @@ def name(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
+    @property
+    def data_type(self):
+        """Retrieves all the data_types the workflow accepts
+
+        Returns
+        ----------
+        list of str
+            The data types
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT data_type
+                     FROM qiita.default_workflow_data_type
+                     LEFT JOIN qiita.data_type USING (data_type_id)
+                     WHERE default_workflow_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchflatten()
+
     @property
     def graph(self):
         """Returns the graph that represents the workflow

qiita_db/support_files/patches/81.sql

@@ -3,3 +3,37 @@
 
 ALTER TABLE qiita.prep_template ADD creation_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP;
 ALTER TABLE qiita.prep_template ADD modification_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP;
+
+
+
+-- Feb 23, 2021
+
+-- a. Removing software_id from qiita.default_workflow and replacing it by a
+--    table which will like different data_types with the default_workflow +
+--    adding an active flag in case we need to deprecate default_workflows
+ALTER TABLE qiita.default_workflow DROP software_id;
+CREATE TABLE qiita.default_workflow_data_type (
+	default_workflow_id	BIGINT NOT NULL,
+	data_type_id BIGINT NOT NULL,
+  CONSTRAINT fk_default_workflow_id FOREIGN KEY ( default_workflow_id ) REFERENCES qiita.default_workflow( default_workflow_id ),
+  CONSTRAINT fk_data_type_id FOREIGN KEY ( data_type_id ) REFERENCES qiita.data_type ( data_type_id ),
+  PRIMARY KEY(default_workflow_id, data_type_id)
+);
+ALTER TABLE qiita.default_workflow ADD active BOOL DEFAULT TRUE;
+
+-- b. Removing command_id from qiita.default_workflow_node and default_parameter_set as this information
+--    can be accessed via the default_parameter object (the info is duplicated)
+ALTER TABLE qiita.default_workflow_node DROP command_id;
+
+-- c. Linking some of the data_types with the default_workflows; note that this
+--    is fine for the test database but we are going to need to clean up and
+--    insert the most up to date recommendations directly in qiita.ucsd.edu
+INSERT INTO qiita.default_workflow_data_type (default_workflow_id, data_type_id) VALUES
+  -- data types:
+  --   1 | 16S
+  --   2 | 18S
+  --   3 | ITS
+  (1, 1),
+  (1, 2),
+  (2, 2),
+  (3, 3);

qiita_db/support_files/qiita-db.dbs

@@ -524,20 +524,27 @@
 		</table>
 		<table name="default_workflow" >
 			<column name="default_workflow_id" type="bigserial" jt="-5" mandatory="y" />
-			<column name="software_id" type="bigint" jt="-5" mandatory="y" />
 			<column name="name" type="varchar" jt="12" mandatory="y" />
+			<column name="active" type="boolean" jt="-7" />
 			<index name="pk_default_workflow" unique="PRIMARY_KEY" >
 				<column name="default_workflow_id" />
 			</index>
 			<index name="idx_default_workflow" unique="UNIQUE_KEY" >
-				<column name="software_id" />
 				<column name="name" />
 			</index>
-			<index name="idx_default_workflow" unique="NORMAL" >
-				<column name="software_id" />
+		</table>
+		<table name="default_workflow_data_type" prior="tbl" >
+			<column name="default_workflow_id" type="bigint" jt="-5" mandatory="y" />
+			<column name="data_type_id" type="bigint" jt="-5" mandatory="y" />
+			<index name="pk_default_workflow_data_type" unique="PRIMARY_KEY" >
+				<column name="default_workflow_id" />
+				<column name="data_type_id" />
 			</index>
-			<fk name="fk_default_workflow_software" to_schema="qiita" to_table="software" >
-				<fk_column name="software_id" pk="software_id" />
+			<fk name="fk_default_workflow_id" to_schema="qiita" to_table="default_workflow" >
+				<fk_column name="default_workflow_id" pk="default_workflow_id" />
+			</fk>
+			<fk name="fk_data_type_id" to_schema="qiita" to_table="data_type" >
+				<fk_column name="data_type_id" pk="data_type_id" />
 			</fk>
 		</table>
 		<table name="default_workflow_edge" >
@@ -591,11 +598,7 @@
 		<table name="default_workflow_node" >
 			<column name="default_workflow_node_id" type="bigserial" jt="-5" mandatory="y" />
 			<column name="default_workflow_id" type="bigint" jt="-5" mandatory="y" />
-			<column name="command_id" type="bigint" jt="-5" mandatory="y" />
 			<column name="default_parameter_set_id" type="bigint" jt="-5" mandatory="y" />
-			<index name="idx_default_workflow_command" unique="NORMAL" >
-				<column name="command_id" />
-			</index>
 			<index name="idx_default_workflow_command" unique="NORMAL" >
 				<column name="default_parameter_set_id" />
 			</index>
@@ -605,9 +608,6 @@
 			<index name="pk_default_workflow_command" unique="PRIMARY_KEY" >
 				<column name="default_workflow_node_id" />
 			</index>
-			<fk name="fk_default_workflow_command" to_schema="qiita" to_table="software_command" >
-				<fk_column name="command_id" pk="command_id" />
-			</fk>
 			<fk name="fk_default_workflow_command_0" to_schema="qiita" to_table="default_parameter_set" >
 				<fk_column name="default_parameter_set_id" pk="default_parameter_set_id" />
 			</fk>
@@ -1696,24 +1696,25 @@ Controlled Vocabulary]]></comment>
 		<entity schema="qiita" name="archive_merging_scheme" color="B2CDF7" x="1248" y="1472" />
 		<entity schema="qiita" name="artifact" color="B2CDF7" x="1200" y="880" />
 		<entity schema="qiita" name="artifact_filepath" color="B2CDF7" x="1040" y="960" />
-		<entity schema="qiita" name="artifact_output_processing_job" color="B2CDF7" x="1760" y="1376" />
-		<entity schema="qiita" name="artifact_processing_job" color="B2CDF7" x="1728" y="1280" />
+		<entity schema="qiita" name="artifact_output_processing_job" color="B2CDF7" x="1760" y="1392" />
+		<entity schema="qiita" name="artifact_processing_job" color="B2CDF7" x="1760" y="1296" />
 		<entity schema="qiita" name="artifact_type" color="D0DEF5" x="1408" y="880" />
 		<entity schema="qiita" name="artifact_type_filepath_type" color="B2CDF7" x="1376" y="736" />
 		<entity schema="qiita" name="checksum_algorithm" color="B2CDF7" x="736" y="960" />
 		<entity schema="qiita" name="column_controlled_vocabularies" color="D0DEF5" x="272" y="1168" />
 		<entity schema="qiita" name="column_ontology" color="D0DEF5" x="272" y="1424" />
-		<entity schema="qiita" name="command_output" color="B2CDF7" x="1728" y="1136" />
-		<entity schema="qiita" name="command_parameter" color="B2CDF7" x="2192" y="960" />
+		<entity schema="qiita" name="command_output" color="B2CDF7" x="1760" y="1152" />
+		<entity schema="qiita" name="command_parameter" color="B2CDF7" x="2192" y="896" />
 		<entity schema="qiita" name="controlled_vocab" color="D0DEF5" x="48" y="1168" />
 		<entity schema="qiita" name="controlled_vocab_values" color="D0DEF5" x="48" y="1296" />
 		<entity schema="qiita" name="data_directory" color="B2CDF7" x="576" y="928" />
 		<entity schema="qiita" name="data_type" color="D0DEF5" x="704" y="1120" />
 		<entity schema="qiita" name="default_parameter_set" color="B2CDF7" x="2416" y="1104" />
-		<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2656" y="1040" />
-		<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2640" y="1232" />
-		<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2384" y="1376" />
-		<entity schema="qiita" name="default_workflow_node" color="B2CDF7" x="2416" y="1232" />
+		<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2672" y="1136" />
+		<entity schema="qiita" name="default_workflow_data_type" color="C1D8EE" x="2640" y="1008" />
+		<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2656" y="1248" />
+		<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2576" y="1408" />
+		<entity schema="qiita" name="default_workflow_node" color="B2CDF7" x="2416" y="1264" />
 		<entity schema="qiita" name="ebi_run_accession" color="B2CDF7" x="1344" y="1104" />
 		<entity schema="qiita" name="environmental_package" color="B2CDF7" x="2176" y="176" />
 		<entity schema="qiita" name="filepath" color="C0D4F3" x="640" y="752" />
@@ -1727,42 +1728,42 @@ Controlled Vocabulary]]></comment>
 		<entity schema="qiita" name="oauth_identifiers" color="B7C8E3" x="2352" y="688" />
 		<entity schema="qiita" name="oauth_software" color="B2CDF7" x="2192" y="688" />
 		<entity schema="qiita" name="ontology" color="D0DEF5" x="576" y="1408" />
-		<entity schema="qiita" name="parameter_artifact_type" color="B2CDF7" x="1728" y="1040" />
+		<entity schema="qiita" name="parameter_artifact_type" color="B2CDF7" x="1760" y="1056" />
 		<entity schema="qiita" name="parent_artifact" color="B2CDF7" x="1040" y="848" />
-		<entity schema="qiita" name="parent_processing_job" color="B2CDF7" x="2016" y="1392" />
+		<entity schema="qiita" name="parent_processing_job" color="B2CDF7" x="2032" y="1376" />
 		<entity schema="qiita" name="per_study_tags" color="B2CDF7" x="2192" y="528" />
 		<entity schema="qiita" name="portal_type" color="C0D4F3" x="768" y="560" />
 		<entity schema="qiita" name="prep_template" color="B2CDF7" x="1232" y="464" />
 		<entity schema="qiita" name="prep_template_filepath" color="B2CDF7" x="1024" y="416" />
-		<entity schema="qiita" name="prep_template_processing_job" color="B2CDF7" x="1728" y="928" />
+		<entity schema="qiita" name="prep_template_processing_job" color="B2CDF7" x="1760" y="928" />
 		<entity schema="qiita" name="prep_template_sample" color="D0DEF5" x="1152" y="304" />
 		<entity schema="qiita" name="prep_y" color="D0DEF5" x="992" y="304" />
-		<entity schema="qiita" name="processing_job" color="B2CDF7" x="1968" y="1072" />
-		<entity schema="qiita" name="processing_job_resource_allocation" color="C1D8EE" x="2656" y="1376" />
-		<entity schema="qiita" name="processing_job_status" color="B2CDF7" x="1728" y="1504" />
-		<entity schema="qiita" name="processing_job_validator" color="B2CDF7" x="2192" y="1392" />
-		<entity schema="qiita" name="processing_job_workflow" color="B2CDF7" x="2192" y="1504" />
-		<entity schema="qiita" name="processing_job_workflow_root" color="B2CDF7" x="2448" y="1504" />
-		<entity schema="qiita" name="publication" color="B2CDF7" x="1936" y="688" />
+		<entity schema="qiita" name="processing_job" color="B2CDF7" x="1984" y="1088" />
+		<entity schema="qiita" name="processing_job_resource_allocation" color="C1D8EE" x="2256" y="1408" />
+		<entity schema="qiita" name="processing_job_status" color="B2CDF7" x="1760" y="1504" />
+		<entity schema="qiita" name="processing_job_validator" color="B2CDF7" x="2224" y="1312" />
+		<entity schema="qiita" name="processing_job_workflow" color="B2CDF7" x="2224" y="1552" />
+		<entity schema="qiita" name="processing_job_workflow_root" color="B2CDF7" x="2496" y="1552" />
+		<entity schema="qiita" name="publication" color="B2CDF7" x="1936" y="656" />
 		<entity schema="qiita" name="qiita_user" color="D0DEF5" x="704" y="288" />
-		<entity schema="qiita" name="reference" color="C0D4F3" x="2016" y="1504" />
+		<entity schema="qiita" name="reference" color="C0D4F3" x="2032" y="1488" />
 		<entity schema="qiita" name="restrictions" color="B2CDF7" x="1584" y="400" />
 		<entity schema="qiita" name="sample_template_filepath" color="B2CDF7" x="1008" y="560" />
 		<entity schema="qiita" name="sample_x" color="D0DEF5" x="1568" y="288" />
 		<entity schema="qiita" name="severity" color="C0D4F3" x="1120" y="1296" />
 		<entity schema="qiita" name="software" color="B2CDF7" x="2432" y="896" />
-		<entity schema="qiita" name="software_artifact_type" color="B2CDF7" x="1984" y="944" />
-		<entity schema="qiita" name="software_command" color="B2CDF7" x="2192" y="1184" />
-		<entity schema="qiita" name="software_publication" color="B2CDF7" x="1920" y="784" />
-		<entity schema="qiita" name="software_type" color="B2CDF7" x="2640" y="912" />
+		<entity schema="qiita" name="software_artifact_type" color="B2CDF7" x="2000" y="976" />
+		<entity schema="qiita" name="software_command" color="B2CDF7" x="2208" y="1104" />
+		<entity schema="qiita" name="software_publication" color="B2CDF7" x="1904" y="768" />
+		<entity schema="qiita" name="software_type" color="B2CDF7" x="2640" y="896" />
 		<entity schema="qiita" name="stats_daily" color="B2CDF7" x="2464" y="400" />
 		<entity schema="qiita" name="study" color="D0DEF5" x="1776" y="208" />
 		<entity schema="qiita" name="study_artifact" color="B2CDF7" x="1600" y="720" />
 		<entity schema="qiita" name="study_environmental_package" color="B2CDF7" x="2176" y="80" />
 		<entity schema="qiita" name="study_person" color="C0D4F3" x="2000" y="224" />
 		<entity schema="qiita" name="study_portal" color="A8C4EF" x="1920" y="80" />
 		<entity schema="qiita" name="study_prep_template" color="D0DEF5" x="1536" y="560" />
-		<entity schema="qiita" name="study_publication" color="B2CDF7" x="1792" y="688" />
+		<entity schema="qiita" name="study_publication" color="B2CDF7" x="1776" y="672" />
 		<entity schema="qiita" name="study_sample" color="D0DEF5" x="1376" y="288" />
 		<entity schema="qiita" name="study_tags" color="B2CDF7" x="2176" y="416" />
 		<entity schema="qiita" name="study_users" color="D0DEF5" x="1776" y="96" />