[tf.learn] High-level DNNAutoencoder (tensorflow#2088)

* Added BaseTransformer and DNNAutoencoder

* Fix conflict and added example

Author: terrytangyuan

tensorflow/contrib/learn/python/learn/estimators/__init__.py

@@ -16,7 +16,7 @@
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.learn.python.learn.estimators.base import TensorFlowEstimator
+from tensorflow.contrib.learn.python.learn.estimators.base import TensorFlowEstimator, TensorFlowBaseTransformer
 from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowLinearClassifier
 from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowClassifier
 from tensorflow.contrib.learn.python.learn.estimators.linear import TensorFlowLinearRegressor
@@ -25,4 +25,5 @@
 from tensorflow.contrib.learn.python.learn.estimators.dnn import TensorFlowDNNRegressor
 from tensorflow.contrib.learn.python.learn.estimators.rnn import TensorFlowRNNClassifier
 from tensorflow.contrib.learn.python.learn.estimators.rnn import TensorFlowRNNRegressor
+from tensorflow.contrib.learn.python.learn.estimators.autoencoder import TensorFlowDNNAutoencoder
 from tensorflow.contrib.learn.python.learn.estimators.run_config import RunConfig

tensorflow/contrib/learn/python/learn/estimators/_sklearn.py

@@ -111,6 +111,8 @@ class _RegressorMixin():
   """Mixin class for all regression estimators."""
   pass
 
+class _TransformerMixin():
+    """Mixin class for all transformer estimators."""
 
 class _NotFittedError(ValueError, AttributeError):
   """Exception class to raise if estimator is used before fitting.
@@ -167,10 +169,11 @@ def _train_test_split(*args, **options):
     result += [x.take(train_idx, axis=0), x.take(test_idx, axis=0)]
   return tuple(result)
 
+
 # If "TENSORFLOW_SKLEARN" flag is defined then try to import from sklearn.
 TRY_IMPORT_SKLEARN = os.environ.get('TENSORFLOW_SKLEARN', False)
 if TRY_IMPORT_SKLEARN:
-  from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
+  from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin, TransformerMixin
   from sklearn.metrics import accuracy_score, log_loss, mean_squared_error
   from sklearn.cross_validation import train_test_split
   try:
@@ -185,6 +188,7 @@ def _train_test_split(*args, **options):
   BaseEstimator = _BaseEstimator
   ClassifierMixin = _ClassifierMixin
   RegressorMixin = _RegressorMixin
+  TransformerMixin = _TransformerMixin
   NotFittedError = _NotFittedError
   accuracy_score = _accuracy_score
   log_loss = None

tensorflow/contrib/learn/python/learn/estimators/autoencoder.py

@@ -0,0 +1,116 @@
+"""Deep Autoencoder estimators."""
+#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import nn
+from tensorflow.contrib.learn.python.learn.estimators.base import TensorFlowBaseTransformer
+from tensorflow.contrib.learn.python.learn import models
+
+
+class TensorFlowDNNAutoencoder(TensorFlowBaseTransformer):
+    """TensorFlow Autoencoder Regressor model.
+
+    Parameters:
+        hidden_units: List of hidden units per layer.
+        batch_size: Mini batch size.
+        activation: activation function used to map inner latent layer onto
+                    reconstruction layer.
+        add_noise: a function that adds noise to tensor_in, 
+               e.g. def add_noise(x):
+                        return(x + np.random.normal(0, 0.1, (len(x), len(x[0]))))
+        steps: Number of steps to run over data.
+        optimizer: Optimizer name (or class), for example "SGD", "Adam",
+                   "Adagrad".
+        learning_rate: If this is constant float value, no decay function is used.
+            Instead, a customized decay function can be passed that accepts
+            global_step as parameter and returns a Tensor.
+            e.g. exponential decay function:
+            def exp_decay(global_step):
+                return tf.train.exponential_decay(
+                    learning_rate=0.1, global_step,
+                    decay_steps=2, decay_rate=0.001)
+        continue_training: when continue_training is True, once initialized
+            model will be continuely trained on every call of fit.
+        config: RunConfig object that controls the configurations of the session,
+            e.g. num_cores, gpu_memory_fraction, etc.
+        verbose: Controls the verbosity, possible values:
+                 0: the algorithm and debug information is muted.
+                 1: trainer prints the progress.
+                 2: log device placement is printed.
+        dropout: When not None, the probability we will drop out a given
+                 coordinate.
+    """
+    def __init__(self, hidden_units, n_classes=0, batch_size=32,
+                 steps=200, optimizer="Adagrad", learning_rate=0.1,
+                 clip_gradients=5.0, activation=nn.relu, add_noise=None,
+                 continue_training=False, config=None,
+                 verbose=1, dropout=None):
+        self.hidden_units = hidden_units
+        self.dropout = dropout
+        self.activation = activation
+        self.add_noise = add_noise
+        super(TensorFlowDNNAutoencoder, self).__init__(
+            model_fn=self._model_fn,
+            n_classes=n_classes,
+            batch_size=batch_size, steps=steps, optimizer=optimizer,
+            learning_rate=learning_rate, clip_gradients=clip_gradients,
+            continue_training=continue_training,
+            config=config, verbose=verbose)
+
+    def _model_fn(self, X, y):
+        encoder, decoder, autoencoder_estimator = models.get_autoencoder_model(
+            self.hidden_units,
+            models.linear_regression,
+            activation=self.activation,
+            add_noise=self.add_noise,
+            dropout=self.dropout)(X)
+        self.encoder = encoder
+        self.decoder = decoder
+        return autoencoder_estimator
+
+    def generate(self, hidden=None):
+        """Generate new data using trained construction layer"""
+        if hidden is None:
+            last_layer = len(self.hidden_units) - 1
+            bias = self.get_tensor_value('encoder/dnn/layer%d/Linear/Bias:0' % last_layer)
+            import numpy as np
+            hidden = np.random.normal(size=bias.shape)
+            hidden = np.reshape(hidden, (1, len(hidden)))
+        return self._session.run(self.decoder, feed_dict={self.encoder: hidden})
+
+    @property
+    def weights_(self):
+        """Returns weights of the autoencoder's weight layers."""
+        weights = []
+        for layer in range(len(self.hidden_units)):
+            weights.append(self.get_tensor_value('encoder/dnn/layer%d/Linear/Matrix:0' % layer))
+        for layer in range(len(self.hidden_units)):
+            weights.append(self.get_tensor_value('decoder/dnn/layer%d/Linear/Matrix:0' % layer))
+        weights.append(self.get_tensor_value('linear_regression/weights:0'))
+        return weights
+
+    @property
+    def bias_(self):
+        """Returns bias of the autoencoder's bias layers."""
+        biases = []
+        for layer in range(len(self.hidden_units)):
+            biases.append(self.get_tensor_value('encoder/dnn/layer%d/Linear/Bias:0' % layer))
+        for layer in range(len(self.hidden_units)):
+            biases.append(self.get_tensor_value('decoder/dnn/layer%d/Linear/Bias:0' % layer))
+        biases.append(self.get_tensor_value('linear_regression/bias:0'))
+        return biases
+

tensorflow/contrib/learn/python/learn/estimators/base.py

@@ -148,7 +148,8 @@ def _setup_training(self):
       # Add histograms for X and y if they are floats.
       if self._data_feeder.input_dtype in (np.float32, np.float64):
         logging_ops.histogram_summary('X', self._inp)
-      if self._data_feeder.output_dtype in (np.float32, np.float64):
+      if self._data_feeder.output_dtype in (np.float32, np.float64)\
+        and self._out is not None:
         logging_ops.histogram_summary('y', self._out)
 
       # Create model's graph.
@@ -408,7 +409,8 @@ def _setup_training(self):
             # Add histograms for X and y if they are floats.
             if self._data_feeder.input_dtype in (np.float32, np.float64):
                 logging_ops.histogram_summary("X", self._inp)
-            if self._data_feeder.output_dtype in (np.float32, np.float64):
+            if self._data_feeder.output_dtype in (np.float32, np.float64)\
+               and self._out is not None:
                 logging_ops.histogram_summary("y", self._out)
 
             # Create model's graph.
@@ -959,3 +961,18 @@ def restore(cls, path, config=None):
     estimator = getattr(estimators, class_name)(**model_def)
     estimator._restore(path)
     return estimator
+
+
+class TensorFlowBaseTransformer(TensorFlowEstimator, _sklearn.TransformerMixin):
+    """TensorFlow Base Transformer class."""
+    def transform(self, X):
+        """Transform X using trained transformer."""
+        return(super(TensorFlowBaseTransformer, self).predict(X, axis=1, batch_size=None))
+
+    def fit(self, X, y=None, monitor=None, logdir=None):
+        """Fit a transformer."""
+        return(super(TensorFlowBaseTransformer, self).fit(X, y, monitor=None, logdir=None))
+
+    def fit_transform(self, X, y=None, monitor=None, logdir=None):
+        """Fit transformer and transform X using trained transformer."""
+        return(self.fit(X, y, monitor=None, logdir=None).transform(X))

tensorflow/contrib/learn/python/learn/models.py

@@ -18,6 +18,7 @@
 
 from tensorflow.contrib.learn.python.learn.ops import dnn_ops
 from tensorflow.contrib.learn.python.learn.ops import losses_ops
+from tensorflow.contrib.learn.python.learn.ops import autoencoder_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops as array_ops_
@@ -187,6 +188,36 @@ def dnn_estimator(X, y):
 
   return dnn_estimator
 
+def get_autoencoder_model(hidden_units, target_predictor_fn,
+                          activation, add_noise=None, dropout=None):
+    """Returns a function that creates a Autoencoder TensorFlow subgraph with given
+    params.
+
+    Args:
+        hidden_units: List of values of hidden units for layers.
+        target_predictor_fn: Function that will predict target from input
+                             features. This can be logistic regression,
+                             linear regression or any other model,
+                             that takes X, y and returns predictions and loss tensors.
+        activation: activation function used to map inner latent layer onto
+                    reconstruction layer.
+        add_noise: a function that adds noise to tensor_in, 
+               e.g. def add_noise(x):
+                        return(x + np.random.normal(0, 0.1, (len(x), len(x[0]))))
+        dropout: When not none, causes dropout regularization to be used,
+                 with the specified probability of removing a given coordinate.
+
+    Returns:
+        A function that creates the subgraph.
+    """
+    def dnn_autoencoder_estimator(X):
+        """Autoencoder estimator with target predictor function on top."""
+        encoder, decoder = autoencoder_ops.dnn_autoencoder(
+          X, hidden_units, activation,
+          add_noise=add_noise, dropout=dropout)
+        return encoder, decoder, target_predictor_fn(X, decoder)
+    return dnn_autoencoder_estimator
+
 ## This will be in Tensorflow 0.7.
 ## TODO(ilblackdragon): Clean this up when it's released
 

tensorflow/contrib/learn/python/learn/ops/__init__.py

@@ -19,6 +19,7 @@
 from tensorflow.contrib.learn.python.learn.ops.array_ops import *
 from tensorflow.contrib.learn.python.learn.ops.conv_ops import *
 from tensorflow.contrib.learn.python.learn.ops.dnn_ops import *
+from tensorflow.contrib.learn.python.learn.ops.autoencoder_ops import *
 from tensorflow.contrib.learn.python.learn.ops.dropout_ops import *
 from tensorflow.contrib.learn.python.learn.ops.embeddings_ops import *
 from tensorflow.contrib.learn.python.learn.ops.losses_ops import *

tensorflow/contrib/learn/python/learn/ops/autoencoder_ops.py

@@ -0,0 +1,56 @@
+"""TensorFlow ops for autoencoder."""
+#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.contrib.learn.python.learn.ops import dnn_ops
+
+
+def dnn_autoencoder(tensor_in, hidden_units,
+                activation=nn.relu, add_noise=None,
+                dropout=None, scope=None):
+    """Creates fully connected autoencoder subgraph.
+
+    Args:
+        tensor_in: tensor or placeholder for input features.
+        hidden_units: list of counts of hidden units in each layer.
+        activation: activation function used to map inner latent layer onto
+                    reconstruction layer.
+        add_noise: a function that adds noise to tensor_in, 
+               e.g. def add_noise(x):
+                        return(x + np.random.normal(0, 0.1, (len(x), len(x[0]))))
+        dropout: if not None, will add a dropout layer with given
+                 probability.
+        scope: the variable scope for this op.
+
+    Returns:
+        Tensors for encoder and decoder.
+    """
+    with vs.variable_op_scope([tensor_in], scope, "autoencoder"):
+        if add_noise is not None:
+                    tensor_in = add_noise(tensor_in)
+        with vs.variable_scope('encoder'):
+            # build DNN encoder
+            encoder = dnn_ops.dnn(tensor_in, hidden_units,
+                activation=activation, dropout=dropout)
+        with vs.variable_scope('decoder'):
+            # reverse hidden_units and built DNN decoder
+            decoder = dnn_ops.dnn(encoder, hidden_units[::-1],
+                activation=activation, dropout=dropout)
+        return encoder, decoder
+

tensorflow/contrib/learn/python/learn/tests/test_nonlinear.py

@@ -165,6 +165,16 @@ def input_fn(X):
                                                                       5, 6]])))
     self.assertAllClose(predictions, np.array([1, 0]))
 
+  def testDNNAutoencoder(self):
+    import numpy as np
+    iris = datasets.load_iris()
+    autoencoder = learn.TensorFlowDNNAutoencoder(hidden_units=[10, 20])
+    transformed = autoencoder.fit_transform(iris.data[1:2])
+    expected = np.array([[ -3.57627869e-07, 1.17000043e+00, 1.01902664e+00, 1.19209290e-07,
+                            0.00000000e+00, 1.19209290e-07, -5.96046448e-08, -2.38418579e-07,
+                            9.74681854e-01, 1.19209290e-07]])
+    self.assertAllClose(transformed, expected)
+
 
 if __name__ == "__main__":
   tf.test.main()

tensorflow/examples/skflow/README.md

@@ -10,6 +10,7 @@ Some examples use the `pandas` library for data processing (`sudo pip install pa
 * [Deep Neural Network Regression with Boston Data](boston.py)
 * [Convolutional Neural Networks with Digits Data](digits.py)
 * [Deep Neural Network Classification with Iris Data](iris.py)
+* [Deep Neural Network Autoencoder with Iris Data](dnn_autoencoder_iris.py)
 * [Grid search and Deep Neural Network Classification](iris_gridsearch_cv.py)
 * [Deep Neural Network with Customized Decay Function](iris_custom_decay_dnn.py)
 * [Building A Custom Model](iris_custom_model.py)

tensorflow/examples/skflow/dnn_autoencoder_iris.py

@@ -0,0 +1,35 @@
+#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import random
+
+import tensorflow as tf
+from tensorflow.contrib.learn.python import learn
+from tensorflow.contrib.learn.python.learn import datasets
+
+# Load Iris Data
+iris = datasets.load_iris()
+
+# Initialize a deep neural network autoencoder
+# You can also add noise and add dropout if needed
+# Details see TensorFlowDNNAutoencoder documentation.
+autoencoder = learn.TensorFlowDNNAutoencoder(hidden_units=[10, 20])
+
+# Fit with Iris data
+transformed = autoencoder.fit_transform(iris.data)
+
+print(transformed)