Merge branch 'tf-transformer-part1' into tf-transformer-part2

Author: phi-dbq

python/sparkdl/param/converters.py

@@ -13,10 +13,15 @@
 # limitations under the License.
 #
 
-# pylint: disable=wrong-spelling-in-docstring,invalid-name,import-error
+# pylint: disable=invalid-name,import-error
 
 """ SparkDLTypeConverters
-Type conversion utilities for definition Spark Deep Learning related MLlib `Params`.
+
+Type conversion utilities for defining MLlib `Params` used in Spark Deep Learning Pipelines.
+
+.. note:: We follow the convention of MLlib to name these utilities "converters",
+          but most of them act as type checkers that return the argument if it is
+          the desired type and raise `TypeError` otherwise.
 """
 
 import six
@@ -33,7 +38,7 @@ class SparkDLTypeConverters(object):
     """
     .. note:: DeveloperApi
 
-    Factory methods for type conversion functions for :py:func:`Param.typeConverter`.
+    Methods for type conversion functions for :py:func:`Param.typeConverter`.
     These methods are similar to :py:class:`spark.ml.param.TypeConverters`.
     They provide support for the `Params` types introduced in Spark Deep Learning Pipelines.
     """
@@ -50,19 +55,16 @@ def toTFGraph(value):
     @staticmethod
     def asColumnToTensorNameMap(value):
         """
-        Convert a value to a column name to :py:obj:`tf.Tensor` name mapping
-        as a sorted list of string pairs, if possible.
+        Convert a value to a column name to :py:class:`tf.Tensor` name mapping
+        as a sorted list (in lexicographical order) of string pairs, if possible.
         """
         if not isinstance(value, dict):
             err_msg = "Could not convert [type {}] {} to column name to tf.Tensor name mapping"
             raise TypeError(err_msg.format(type(value), value))
 
-        # Conversion logic after quick type check
         strs_pair_seq = []
         for _maybe_col_name, _maybe_tnsr_name in value.items():
-            # Check if the non-tensor value is of string type
             _check_is_str(_maybe_col_name)
-            # Check if the tensor name looks like a tensor name
             _check_is_tensor_name(_maybe_tnsr_name)
             strs_pair_seq.append((_maybe_col_name, _maybe_tnsr_name))
 
@@ -71,52 +73,55 @@ def asColumnToTensorNameMap(value):
     @staticmethod
     def asTensorNameToColumnMap(value):
         """
-        Convert a value to a :py:obj:`tf.Tensor` name to column name mapping
-        as a sorted list of string pairs, if possible.
+        Convert a value to a :py:class:`tf.Tensor` name to column name mapping
+        as a sorted list (in lexicographical order) of string pairs, if possible.
         """
         if not isinstance(value, dict):
             err_msg = "Could not convert [type {}] {} to tf.Tensor name to column name mapping"
             raise TypeError(err_msg.format(type(value), value))
 
-        # Conversion logic after quick type check
         strs_pair_seq = []
         for _maybe_tnsr_name, _maybe_col_name in value.items():
-            # Check if the non-tensor value is of string type
             _check_is_str(_maybe_col_name)
-            # Check if the tensor name looks like a tensor name
             _check_is_tensor_name(_maybe_tnsr_name)
             strs_pair_seq.append((_maybe_tnsr_name, _maybe_col_name))
 
         return sorted(strs_pair_seq)
 
     @staticmethod
     def toTFHParams(value):
-        """ Convert a value to a :py:obj:`tf.contrib.training.HParams` object, if possible. """
+        """
+        Check that the given value is a :py:class:`tf.contrib.training.HParams` object,
+        and return it. Raise an error otherwise.
+        """
         if not isinstance(value, tf.contrib.training.HParams):
             raise TypeError("Could not convert %s to TensorFlow HParams" % type(value))
 
         return value
 
     @staticmethod
     def toTFTensorName(value):
-        """ Convert a value to a :py:obj:`tf.Tensor` name, if possible. """
+        """
+        Check if a value is a valid :py:class:`tf.Tensor` name and return it.
+        Raise an error otherwise.
+        """
         if isinstance(value, tf.Tensor):
             return value.name
         try:
-            _maybe_tnsr_name = TypeConverters.toString(value)
-            _check_is_tensor_name(_maybe_tnsr_name)
-            return _maybe_tnsr_name
+            _check_is_tensor_name(value)
+            return value
         except Exception as exc:
             err_msg = "Could not convert [type {}] {} to tf.Tensor name. {}"
             raise TypeError(err_msg.format(type(value), value, exc))
 
     @staticmethod
-    def buildCheckList(supportedList):
+    def buildSupportedItemConverter(supportedList):
         """
-        Create a converter that try to check if a value is part of the supported list.
+        Create a "converter" that try to check if a value is part of the supported list of values.
 
         :param supportedList: list, containing supported objects.
-        :return: a converter that try to convert a value if it is part of the `supportedList`.
+        :return: a converter that try to check if a value is part of the `supportedList` and return it.
+                 Raise an error otherwise.
         """
 
         def converter(value):
@@ -131,7 +136,10 @@ def converter(value):
 
     @staticmethod
     def toKerasLoss(value):
-        """ Convert a value to a name of Keras loss function, if possible """
+        """
+        Check if a value is a valid Keras loss function name and return it.
+        Otherwise raise an error.
+        """
         # return early in for clarify as well as less indentation
         if not kmutil.is_valid_loss_function(value):
             err_msg = "Named loss not supported in Keras: [type {}] {}"
@@ -141,7 +149,10 @@ def toKerasLoss(value):
 
     @staticmethod
     def toKerasOptimizer(value):
-        """ Convert a value to a name of Keras optimizer, if possible """
+        """
+        Check if a value is a valid name of Keras optimizer and return it.
+        Otherwise raise an error.
+        """
         if not kmutil.is_valid_optimizer(value):
             err_msg = "Named optimizer not supported in Keras: [type {}] {}"
             raise TypeError(err_msg.format(type(value), value))
@@ -150,7 +161,7 @@ def toKerasOptimizer(value):
 
 
 def _check_is_tensor_name(_maybe_tnsr_name):
-    """ Check if the input is a valid tensor name """
+    """ Check if the input is a valid tensor name or raise a `TypeError` otherwise. """
     if not isinstance(_maybe_tnsr_name, six.string_types):
         err_msg = "expect tensor name to be of string type, but got [type {}]"
         raise TypeError(err_msg.format(type(_maybe_tnsr_name)))
@@ -164,13 +175,10 @@ def _check_is_tensor_name(_maybe_tnsr_name):
         err_msg = "Tensor name must be of type <op_name>:<index>, but got {}"
         raise TypeError(err_msg.format(_maybe_tnsr_name))
 
-    return _maybe_tnsr_name
-
 
-def _check_is_str(_maybe_col_name):
-    """ Check if the given colunm name is a valid column name """
+def _check_is_str(_maybe_str):
+    """ Check if the value is a valid string type or raise a `TypeError` otherwise. """
     # We only check if the column name candidate is a string type
-    if not isinstance(_maybe_col_name, six.string_types):
+    if not isinstance(_maybe_str, six.string_types):
         err_msg = 'expect string type but got type {} for {}'
-        raise TypeError(err_msg.format(type(_maybe_col_name), _maybe_col_name))
-    return _maybe_col_name
+        raise TypeError(err_msg.format(type(_maybe_str), _maybe_str))

python/sparkdl/param/image_params.py

@@ -107,7 +107,7 @@ class HasOutputMode(Params):
                        "How the output column should be formatted. 'vector' for a 1-d MLlib " +
                        "Vector of floats. 'image' to format the output to work with the image " +
                        "tools in this package.",
-                       typeConverter=SparkDLTypeConverters.buildCheckList(OUTPUT_MODES))
+                       typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(OUTPUT_MODES))
 
     def setOutputMode(self, value):
         return self._set(outputMode=value)

python/sparkdl/param/shared_params.py

@@ -232,7 +232,7 @@ class HasTFHParams(Params):
                       key-value object, storing parameters to be used to define the final
                       TensorFlow graph for the Transformer.
 
-                      Currently accepted values are:
+                      Currently used values are:
                       - `batch_size`: number of samples evaluated together in inference steps"""),
                       typeConverter=SparkDLTypeConverters.toTFHParams)
 

python/sparkdl/transformers/named_image.py

@@ -40,7 +40,7 @@ class DeepImagePredictor(Transformer, HasInputCol, HasOutputCol):
     """
 
     modelName = Param(Params._dummy(), "modelName", "A deep learning model name",
-                      typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS))
+                      typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS))
     decodePredictions = Param(Params._dummy(), "decodePredictions",
                               "If true, output predictions in the (class, description, probability) format",
                               typeConverter=TypeConverters.toBoolean)
@@ -125,7 +125,7 @@ class DeepImageFeaturizer(Transformer, HasInputCol, HasOutputCol):
     """
 
     modelName = Param(Params._dummy(), "modelName", "A deep learning model name",
-                      typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS))
+                      typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS))
 
     @keyword_only
     def __init__(self, inputCol=None, outputCol=None, modelName=None):
@@ -169,7 +169,7 @@ class _NamedImageTransformer(Transformer, HasInputCol, HasOutputCol):
     """
 
     modelName = Param(Params._dummy(), "modelName", "A deep learning model name",
-                      typeConverter=SparkDLTypeConverters.buildCheckList(SUPPORTED_MODELS))
+                      typeConverter=SparkDLTypeConverters.buildSupportedItemConverter(SUPPORTED_MODELS))
     featurize = Param(Params._dummy(), "featurize",
                       "If true, output features. If false, output predictions. Either way the output is a vector.",
                       typeConverter=TypeConverters.toBoolean)