Improve workflows (#3083)

* fix #3076

* addressing @ElDeveloper comments

* adding workflows GUI

* improve HTML

* db changes and improvements

* comment out new db changes

* rm default_workflows from PluginHandler.get

* improving workflows after deployment

* fix error

* clean up

* minor fixes

* improving html

* comments from @justinshaffer

* addressing @ElDeveloper comments

Author: antgonza

qiita_db/processing_job.py

@@ -1974,7 +1974,7 @@ def from_default_workflow(cls, user, dflt_wf, req_params, name=None,
                     parent_ids.append(source_id)
                     # Get the connections between the job and the source
                     connections = data['connections'].connections
-                    for out, in_param in connections:
+                    for out, in_param, _ in connections:
                         # We take advantage of the fact the parameters are
                         # stored in JSON to encode the name of the output
                         # artifact from the previous job

qiita_db/software.py

@@ -1803,12 +1803,13 @@ def connections(self):
             the destination command.
         """
         with qdb.sql_connection.TRN:
-            sql = """SELECT name, parameter_name
+            sql = """SELECT name, parameter_name, artifact_type
                      FROM qiita.default_workflow_edge_connections c
                         JOIN qiita.command_output o
                             ON c.parent_output_id = o.command_output_id
                         JOIN qiita.command_parameter p
                             ON c.child_input_id = p.command_parameter_id
+                        LEFT JOIN qiita.artifact_type USING (artifact_type_id)
                      WHERE default_workflow_edge_id = %s"""
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchindex()
@@ -1852,8 +1853,8 @@ def iter(cls, active=True):
     def active(self):
         """Retrieves active status of the default workflow
 
-        Retruns
-        ----------
+        Returns
+        -------
         active : bool
             active value
         """
@@ -1886,9 +1887,38 @@ def name(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
+    @property
+    def description(self):
+        """Retrieves the description of the default workflow
+
+        Returns
+        -------
+        str
+            description value
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT description
+                     FROM qiita.default_workflow
+                     WHERE default_workflow_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
+    @description.setter
+    def description(self, description):
+        """Changes the description of the default workflow
+
+        Parameters
+        ----------
+        description : str
+            New description value
+        """
+        sql = """UPDATE qiita.default_workflow SET description = %s
+                 WHERE default_workflow_id = %s"""
+        qdb.sql_connection.perform_as_transaction(sql, [description, self._id])
+
     @property
     def data_type(self):
-        """Retrieves all the data_types the workflow accepts
+        """Retrieves all the data_types accepted by the default workflow
 
         Returns
         ----------
@@ -1917,7 +1947,8 @@ def graph(self):
             # Retrieve all graph workflow nodes
             sql = """SELECT default_workflow_node_id
                      FROM qiita.default_workflow_node
-                     WHERE default_workflow_id = %s"""
+                     WHERE default_workflow_id = %s
+                     ORDER BY default_workflow_node_id"""
             qdb.sql_connection.TRN.add(sql, [self.id])
             db_nodes = qdb.sql_connection.TRN.execute_fetchflatten()
 
@@ -1930,7 +1961,8 @@ def graph(self):
                         JOIN qiita.default_workflow_node n
                             ON e.parent_id = n.default_workflow_node_id
                             OR e.child_id = n.default_workflow_node_id
-                     WHERE default_workflow_id = %s"""
+                     WHERE default_workflow_id = %s
+                     ORDER BY default_workflow_edge_id"""
             qdb.sql_connection.TRN.add(sql, [self.id])
             db_edges = qdb.sql_connection.TRN.execute_fetchindex()
 

qiita_db/support_files/patches/81.sql

@@ -10,7 +10,8 @@ ALTER TABLE qiita.prep_template ADD modification_timestamp TIMESTAMP DEFAULT CUR
 
 -- a. Removing software_id from qiita.default_workflow and replacing it by a
 --    table which will like different data_types with the default_workflow +
---    adding an active flag in case we need to deprecate default_workflows
+--    adding an active flag in case we need to deprecate default_workflows +
+--    adding a description column
 ALTER TABLE qiita.default_workflow DROP software_id;
 CREATE TABLE qiita.default_workflow_data_type (
 	default_workflow_id	BIGINT NOT NULL,
@@ -20,6 +21,7 @@ CREATE TABLE qiita.default_workflow_data_type (
   PRIMARY KEY(default_workflow_id, data_type_id)
 );
 ALTER TABLE qiita.default_workflow ADD active BOOL DEFAULT TRUE;
+ALTER TABLE qiita.default_workflow ADD description TEXT;
 
 -- b. Removing command_id from qiita.default_workflow_node and default_parameter_set as this information
 --    can be accessed via the default_parameter object (the info is duplicated)
@@ -37,3 +39,11 @@ INSERT INTO qiita.default_workflow_data_type (default_workflow_id, data_type_id)
   (1, 2),
   (2, 2),
   (3, 3);
+
+-- d. adding descriptions
+UPDATE qiita.default_workflow
+	SET description = 'This accepts html <a href="https://qiita.ucsd.edu">Qiita!</a><br/><br/><b>BYE!</b>'
+	WHERE default_workflow_id = 1;
+UPDATE qiita.default_workflow
+	SET description = 'This is another description'
+	WHERE default_workflow_id = 2;

qiita_db/support_files/qiita-db.dbs

@@ -526,6 +526,7 @@
 			<column name="default_workflow_id" type="bigserial" jt="-5" mandatory="y" />
 			<column name="name" type="varchar" jt="12" mandatory="y" />
 			<column name="active" type="boolean" jt="-7" />
+			<column name="description" type="text" jt="12" />
 			<index name="pk_default_workflow" unique="PRIMARY_KEY" >
 				<column name="default_workflow_id" />
 			</index>
@@ -1710,7 +1711,7 @@ Controlled Vocabulary]]></comment>
 		<entity schema="qiita" name="data_directory" color="B2CDF7" x="576" y="928" />
 		<entity schema="qiita" name="data_type" color="D0DEF5" x="704" y="1120" />
 		<entity schema="qiita" name="default_parameter_set" color="B2CDF7" x="2416" y="1104" />
-		<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2672" y="1136" />
+		<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2656" y="1120" />
 		<entity schema="qiita" name="default_workflow_data_type" color="C1D8EE" x="2640" y="1008" />
 		<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2656" y="1248" />
 		<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2576" y="1408" />

qiita_db/support_files/qiita-db.html

@@ -2,7 +2,7 @@
 <!DOCTYPE html>
 <html lang='en-us' xmlns='http://www.w3.org/1999/xhtml' >
 <head>
-<title>qiita&#45;db</title>
+<title>qiita</title>
 <meta http-equiv='Content-Type' content='text/html; charset=UTF-8'/>
 <meta name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no' />
 <link rel='shortcut icon' href='https://dbschema.com/images/favicon.ico'>
@@ -299,7 +299,7 @@
 <!-- == Legend == -->
 <g transform='translate(10,10)'>
   <rect x='10' y='10' width='230' height='56' rx='6' ry='6' style='fill:url(#legendGradient);filter:url(#shadow);'/> 
-  <text x='20' y='25' class='legendTitle' >qiita-db</text> 
+  <text x='20' y='25' class='legendTitle' >qiita</text> 
   <a xlink:href='https://dbschema.com'><text x='20' y='39' class='legendSubTitle'><tspan>(c) DbSchema</tspan></text></a> 
   <text x='20' y='53' class='legendSubTitle'><tspan>Hover columns to read the comments.</tspan></text> 
 </g>
@@ -629,13 +629,13 @@
 default_parameter_set ref software_command ( command_id )</title></path>
 <text x='2445' y='1218' transform='rotate(90 2445,1218)' class='relName'>command_id</text>
 <!-- == Fk 'default_workflow_data_type_fk_default_workflow_id' == -->
-<path id='default_workflow_data_type_fk_default_workflow_id'  onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" transform='translate(8,0)' class='scene' d='M 2688,1088L 2688,1120' >
+<path id='default_workflow_data_type_fk_default_workflow_id'  onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" transform='translate(8,0)' class='scene' d='M 2672,1088L 2672,1104' >
 	<title>Fk fk_default_workflow_id
 default_workflow_data_type ref default_workflow ( default_workflow_id )</title>
 </path>
-<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)'    d='M 2688,1088L 2688,1120' ><title>Fk fk_default_workflow_id
+<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)'    d='M 2672,1088L 2672,1104' ><title>Fk fk_default_workflow_id
 default_workflow_data_type ref default_workflow ( default_workflow_id )</title></path>
-<text x='2701' y='1090' transform='rotate(90 2701,1090)' class='relName'>default_workflow_id</text>
+<text x='2685' y='1090' transform='rotate(90 2685,1090)' class='relName'>default_workflow_id</text>
 <!-- == Fk 'default_workflow_edge_fk_default_workflow_edge' == -->
 <path id='default_workflow_edge_fk_default_workflow_edge'  onmouseover="hghl(['default_workflow_edge_fk_default_workflow_edge','qiita.default_workflow_edge.parent_id','qiita.default_workflow_node.default_workflow_node_id'])" onmouseout="uhghl(['default_workflow_edge_fk_default_workflow_edge','qiita.default_workflow_edge.parent_id','qiita.default_workflow_node.default_workflow_node_id'])" transform='translate(8,0)' class='scene' d='M 2640,1280L 2624,1280' >
 	<title>Fk fk_default_workflow_edge
@@ -1481,12 +1481,12 @@
 <path d='M 704 1140 L 704 1120 Q 704 1112 712 1112 L 824 1112 Q 832 1112 832 1120 L 832 1140 L704 1140 ' style='fill:url(#tbg_D0DEF5);stroke:1;stroke-opacity:0.1;' />
 <rect class='entity' x='704' y='1112' width='128' height='80' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
 <a xlink:href='#qiita.data_type'><text x='732' y='1131' >data_type</text><title>Table qiita.data_type</title></a>
-  <use id='nn' x='706' y='1149' xlink:href='#nn'/><use id='pk' x='706' y='1148' xlink:href='#pk'><title>Pk pk_data_type ( data_type_id ) </title></use><a xlink:href='#qiita.data_type_data_type_id'><text id='qiita.data_type.data_type_id' x='723' y='1159' onmouseover="hghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id'])" onmouseout="uhghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id'])">data_type_id</text><title>data_type_id
+  <use id='nn' x='706' y='1149' xlink:href='#nn'/><use id='pk' x='706' y='1148' xlink:href='#pk'><title>Pk pk_data_type ( data_type_id ) </title></use><a xlink:href='#qiita.data_type_data_type_id'><text id='qiita.data_type.data_type_id' x='723' y='1159' onmouseover="hghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id'])" onmouseout="uhghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id'])">data_type_id</text><title>data_type_id
 * bigserial</title></a>
 <a xlink:href='#qiita.data_type.data_type_id'><use id='ref' x='820' y='1148' xlink:href='#ref'/><title>Referred by analysis_filepath ( data_type_id ) 
 Referred by artifact ( data_type_id ) 
-Referred by prep_template ( data_type_id ) 
-Referred by default_workflow_data_type ( data_type_id ) </title></a>
+Referred by default_workflow_data_type ( data_type_id ) 
+Referred by prep_template ( data_type_id ) </title></a>
   <use id='nn' x='706' y='1165' xlink:href='#nn'/><use id='unq' x='706' y='1164' xlink:href='#unq'><title>Unq idx_data_type ( data_type ) </title></use><a xlink:href='#qiita.data_type_data_type'><text id='qiita.data_type.data_type' x='723' y='1175'>data_type</text><title>data_type
 * varchar
 Data type (16S, metabolome, etc) the job will use</title></a>
@@ -1509,19 +1509,21 @@
 This is a varchar here - but is of type JSON in postgresql.</title></a>
 <text x='2621' y='1188' text-anchor='end' class='colType'>t</text>
 <!-- == Table 'default_workflow' == -->
-<rect class='entity' style='stroke:none;' x='2672' y='1128' width='176' height='96' rx='8' ry='8' />
-<path d='M 2672 1156 L 2672 1136 Q 2672 1128 2680 1128 L 2840 1128 Q 2848 1128 2848 1136 L 2848 1156 L2672 1156 ' style='fill:url(#tbg_B2CDF7);stroke:1;stroke-opacity:0.1;' />
-<rect class='entity' x='2672' y='1128' width='176' height='96' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
-<a xlink:href='#qiita.default_workflow'><text x='2701' y='1147' >default_workflow</text><title>Table qiita.default_workflow</title></a>
-  <use id='nn' x='2674' y='1165' xlink:href='#nn'/><use id='pk' x='2674' y='1164' xlink:href='#pk'><title>Pk pk_default_workflow ( default_workflow_id ) </title></use><a xlink:href='#qiita.default_workflow_default_workflow_id'><text id='qiita.default_workflow.default_workflow_id' x='2691' y='1175' onmouseover="hghl(['default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id','default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id'])" onmouseout="uhghl(['default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id','default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id'])">default_workflow_id</text><title>default_workflow_id
+<rect class='entity' style='stroke:none;' x='2656' y='1112' width='176' height='112' rx='8' ry='8' />
+<path d='M 2656 1140 L 2656 1120 Q 2656 1112 2664 1112 L 2824 1112 Q 2832 1112 2832 1120 L 2832 1140 L2656 1140 ' style='fill:url(#tbg_B2CDF7);stroke:1;stroke-opacity:0.1;' />
+<rect class='entity' x='2656' y='1112' width='176' height='112' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
+<a xlink:href='#qiita.default_workflow'><text x='2685' y='1131' >default_workflow</text><title>Table qiita.default_workflow</title></a>
+  <use id='nn' x='2658' y='1149' xlink:href='#nn'/><use id='pk' x='2658' y='1148' xlink:href='#pk'><title>Pk pk_default_workflow ( default_workflow_id ) </title></use><a xlink:href='#qiita.default_workflow_default_workflow_id'><text id='qiita.default_workflow.default_workflow_id' x='2675' y='1159' onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id'])">default_workflow_id</text><title>default_workflow_id
 * bigserial</title></a>
-<a xlink:href='#qiita.default_workflow.default_workflow_id'><use id='ref' x='2836' y='1164' xlink:href='#ref'/><title>Referred by default_workflow_node ( default_workflow_id ) 
-Referred by default_workflow_data_type ( default_workflow_id ) </title></a>
-  <use id='nn' x='2674' y='1181' xlink:href='#nn'/><use id='unq' x='2674' y='1180' xlink:href='#unq'><title>Unq idx_default_workflow ( name ) </title></use><a xlink:href='#qiita.default_workflow_name'><text id='qiita.default_workflow.name' x='2691' y='1191'>name</text><title>name
+<a xlink:href='#qiita.default_workflow.default_workflow_id'><use id='ref' x='2820' y='1148' xlink:href='#ref'/><title>Referred by default_workflow_data_type ( default_workflow_id ) 
+Referred by default_workflow_node ( default_workflow_id ) </title></a>
+  <use id='nn' x='2658' y='1165' xlink:href='#nn'/><use id='unq' x='2658' y='1164' xlink:href='#unq'><title>Unq idx_default_workflow ( name ) </title></use><a xlink:href='#qiita.default_workflow_name'><text id='qiita.default_workflow.name' x='2675' y='1175'>name</text><title>name
 * varchar</title></a>
-<text x='2845' y='1188' text-anchor='end' class='colType'>t</text>  <a xlink:href='#qiita.default_workflow_active'><text id='qiita.default_workflow.active' x='2691' y='1207'>active</text><title>active
+<text x='2829' y='1172' text-anchor='end' class='colType'>t</text>  <a xlink:href='#qiita.default_workflow_active'><text id='qiita.default_workflow.active' x='2675' y='1191'>active</text><title>active
 boolean</title></a>
-<text x='2845' y='1204' text-anchor='end' class='colType'>b</text>
+<text x='2829' y='1188' text-anchor='end' class='colType'>b</text>  <a xlink:href='#qiita.default_workflow_description'><text id='qiita.default_workflow.description' x='2675' y='1207'>description</text><title>description
+text</title></a>
+<text x='2829' y='1204' text-anchor='end' class='colType'>t</text>
 <!-- == Table 'default_workflow_data_type' == -->
 <rect class='entity' style='stroke:none;' x='2640' y='1000' width='208' height='80' rx='8' ry='8' />
 <path d='M 2640 1028 L 2640 1008 Q 2640 1000 2648 1000 L 2840 1000 Q 2848 1000 2848 1008 L 2848 1028 L2640 1028 ' style='fill:url(#tbg_C1D8EE);stroke:1;stroke-opacity:0.1;' />
@@ -3696,7 +3698,7 @@
 </table></div></div>
 
 <br><br>
-<div class='card'><div class='card-body'><a name='qiita.default_workflow' onclick='window.scrollTo(2472, 1136);return false;' style='cursor:pointer;'><h5 class='card-title'>Table default_workflow</h5></a>
+<div class='card'><div class='card-body'><a name='qiita.default_workflow' onclick='window.scrollTo(2456, 1120);return false;' style='cursor:pointer;'><h5 class='card-title'>Table default_workflow</h5></a>
 <table class='table table-sm table-bordered small' style='table-layout: fixed; word-wrap: break-word;'>
 <thead>
 <tr><th style='width:5%'>Idx</th><th style='width:40%'>Field Name</th><th>Data Type</th></tr>
@@ -3717,6 +3719,11 @@
 		<td><a name='qiita.default_workflow_active'>active</a></td>
 		<td> boolean  </td>
 	</tr>
+	<tr>
+		<td>&nbsp;</td>
+		<td><a name='qiita.default_workflow_description'>description</a></td>
+		<td> text  </td>
+	</tr>
 <tr><th colspan='3'>Indexes</th></tr>
 	<tr>		<td><svg width='14' height='14'><use xlink:href='#pk'/></svg></td><td>pk&#95;default&#95;workflow</td>
 		<td> ON default&#95;workflow&#95;id</td>