Back-end code cleanup, docs, style (#91)

* chore: Change mentions of 'React' to generic 'ui-graph'

* fix: Change node update from POST->PATCH to be RESTful

* doc: Update/add docstrings for Nodes

* fix: Update remaining Nodes to use flow_vars instead of `**self.options`

* chore: Remove unused import statements

* style: Update spacing, long line lengths

* fix: Update (de)serialization of Workflow to include all attributes

Slight modification to make method `to_json()` instead of `to_session_dict()`. Should be functionally the same.

* style: Re-order method to group be by similar actions

Alphabetical by group. Makes it easier to find, `add_edge` for example.

* chore: Remove old TODOs that have been completed

* fix: Update remaining Workflow calls to `to_json()`

* test: Update to use new to_json method

* chore: Update coverage badges

---------

Co-authored-by: Matt Thomas <matthew_thomas@hms.harvard.edu>

Author: reelmatt

back-end/pyworkflow/pyworkflow/node.py

@@ -48,8 +48,6 @@ def get_execution_options(self, workflow, flow_nodes):
         """
         execution_options = dict()
 
-        # TODO: Can we iterate through flow_vars instead?
-        #       If none are included, we can just return `self.options`.
         for key, option in self.options.items():
 
             if key in flow_nodes:

back-end/pyworkflow/pyworkflow/node_factory.py

@@ -3,8 +3,7 @@
 
 
 def node_factory(node_info):
-    # Create a new Node with info
-    # TODO: should perform error-checking or add default values if missing
+    """Create a new Node with info."""
     node_type = node_info.get('node_type')
     node_key = node_info.get('node_key')
 

back-end/pyworkflow/pyworkflow/nodes/flow_control/integer_input.py

@@ -1,11 +1,11 @@
-from pyworkflow.node import FlowNode, NodeException
+from pyworkflow.node import FlowNode
 from pyworkflow.parameters import *
 
 
 class IntegerNode(FlowNode):
-    """StringNode object
+    """IntegerNode object
 
-    Allows for Strings to replace 'string' fields in Nodes
+    Allows for Integers to replace fields representing numbers in Nodes
     """
     name = "Integer Input"
     num_in = 0

back-end/pyworkflow/pyworkflow/nodes/io/read_csv.py

@@ -5,13 +5,10 @@
 
 
 class ReadCsvNode(IONode):
-    """ReadCsvNode
-
-    Reads a CSV file into a pandas DataFrame.
+    """Reads a CSV file into a pandas DataFrame.
 
     Raises:
-         NodeException: any error reading CSV file, converting
-            to DataFrame.
+         NodeException: any error reading CSV file, converting to DataFrame.
     """
     name = "Read CSV"
     num_in = 0

back-end/pyworkflow/pyworkflow/nodes/io/write_csv.py

@@ -5,13 +5,10 @@
 
 
 class WriteCsvNode(IONode):
-    """WriteCsvNode
-
-    Writes the current DataFrame to a CSV file.
+    """Writes the current DataFrame to a CSV file.
 
     Raises:
-        NodeException: any error writing CSV file, converting
-            from DataFrame.
+        NodeException: any error writing CSV file, converting from DataFrame.
     """
     name = "Write CSV"
     num_in = 1

back-end/pyworkflow/pyworkflow/nodes/manipulation/filter.py

@@ -5,6 +5,14 @@
 
 
 class FilterNode(ManipulationNode):
+    """Subset the DataFrame rows or columns according to the specified index labels.
+
+    pandas API reference:
+    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.filter.html
+
+    Raises:
+        NodeException: catches exceptions when dealing with pandas DataFrames.
+    """
     name = "Filter"
     num_in = 1
     num_out = 1
@@ -31,7 +39,13 @@ class FilterNode(ManipulationNode):
     def execute(self, predecessor_data, flow_vars):
         try:
             input_df = pd.DataFrame.from_dict(predecessor_data[0])
-            output_df = pd.DataFrame.filter(input_df, **self.options)
+            output_df = pd.DataFrame.filter(
+                input_df,
+                items=flow_vars['items'].get_value(),
+                like=flow_vars['like'].get_value(),
+                regex=flow_vars['regex'].get_value(),
+                axis=flow_vars['axis'].get_value(),
+            )
             return output_df.to_json()
         except Exception as e:
             raise NodeException('filter', str(e))

back-end/pyworkflow/pyworkflow/nodes/manipulation/join.py

@@ -5,12 +5,23 @@
 
 
 class JoinNode(ManipulationNode):
+    """Merge DataFrame or named Series objects with a database-style join.
+
+    pandas API reference:
+    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.merge.html
+
+    Raises:
+        NodeException: catches exceptions when dealing with pandas DataFrames.
+    """
     name = "Joiner"
     num_in = 2
     num_out = 1
 
     OPTIONS = {
-        "on": StringParameter("Join Column", docstring="Name of column to join on")
+        "on": StringParameter(
+            "Join Column",
+            docstring="Name of column to join on"
+        )
     }
 
     def execute(self, predecessor_data, flow_vars):

back-end/pyworkflow/pyworkflow/nodes/manipulation/pivot.py

@@ -5,6 +5,14 @@
 
 
 class PivotNode(ManipulationNode):
+    """Create a spreadsheet-style pivot table as a DataFrame.
+
+    pandas reference:
+    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.pivot_table.html
+
+    Raises:
+        NodeException: catches exceptions when dealing with pandas DataFrames.
+    """
     name = "Pivoting"
     num_in = 1
     num_out = 3
@@ -56,7 +64,18 @@ class PivotNode(ManipulationNode):
     def execute(self, predecessor_data, flow_vars):
         try:
             input_df = pd.DataFrame.from_dict(predecessor_data[0])
-            output_df = pd.DataFrame.pivot_table(input_df, **self.options)
+            output_df = pd.DataFrame.pivot_table(
+                input_df,
+                index=flow_vars['index'].get_value(),
+                values=flow_vars['values'].get_value(),
+                columns=flow_vars['columns'].get_value(),
+                aggfunc=flow_vars['aggfunc'].get_value(),
+                fill_value=flow_vars['fill_value'].get_value(),
+                margins=flow_vars['margins'].get_value(),
+                dropna=flow_vars['dropna'].get_value(),
+                margins_name=flow_vars['margins_name'].get_value(),
+                observed=flow_vars['observed'].get_value(),
+            )
             return output_df.to_json()
         except Exception as e:
             raise NodeException('pivot', str(e))

back-end/pyworkflow/pyworkflow/tests/test_workflow.py

@@ -53,7 +53,7 @@ def test_workflow_filename(self):
 
     def test_workflow_from_json(self):
         new_workflow = Workflow("Untitled", root_dir="/tmp")
-        workflow_copy = Workflow.from_json(self.workflow.to_session_dict())
+        workflow_copy = Workflow.from_json(self.workflow.to_json())
 
         self.assertEqual(new_workflow.name, workflow_copy.name)
 
@@ -62,16 +62,17 @@ def test_workflow_from_json_key_error(self):
             new_workflow = Workflow.from_json(dict())
 
     def test_empty_workflow_to_session(self):
-        new_workflow = Workflow("Untitled", root_dir="/tmp")
-        saved_workflow = new_workflow.to_session_dict()
+        new_workflow = Workflow("Untitled", root_dir="/tmp", node_dir=os.path.join(os.getcwd(), 'nodes'))
+        saved_workflow = new_workflow.to_json()
 
         workflow_to_compare = {
             'name': 'Untitled',
             'root_dir': '/tmp',
+            'node_dir': os.path.join(os.getcwd(), 'nodes'),
             'graph': Workflow.to_graph_json(new_workflow.graph),
             'flow_vars': Workflow.to_graph_json(new_workflow.flow_vars),
         }
-        self.assertDictEqual(new_workflow.to_session_dict(), workflow_to_compare)
+        self.assertDictEqual(new_workflow.to_json(), workflow_to_compare)
 
     ##########################
     # Node lists