Keras-like API Advanced Activations, dropout and noise layers

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/ELU.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.{DenseTensorApply, Tensor, TensorFunc6}
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -32,7 +32,7 @@ class ELU[T: ClassTag](
   val alpha: Double = 1.0,
   val inplace: Boolean = false)(
   implicit ev: TensorNumeric[T])
-  extends TensorModule[T]  {
+  extends TensorModule[T] with IdentityOutputShape {
 
   val _alpha = ev.fromType[Double](alpha)
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/GaussianDropout.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -37,7 +37,7 @@ import scala.reflect.ClassTag
 @SerialVersionUID(- 1575781981601306833L)
 class GaussianDropout[T: ClassTag](
    val rate: Double
-  )(implicit ev: TensorNumeric[T]) extends TensorModule[T]{
+  )(implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape{
 
   require(rate < 1 && rate >= 0, s"rate should be in range [0,1)")
   val stddev: Double = Math.sqrt(rate / (1.0-rate))

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/GaussianNoise.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -38,7 +38,7 @@ import scala.reflect.ClassTag
 @SerialVersionUID(- 2590701089601246637L)
 class GaussianNoise[T: ClassTag](
    val stddev: Double
-  )(implicit ev: TensorNumeric[T]) extends TensorModule[T]{
+  )(implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape{
 
 
   override def updateOutput(input: Tensor[T]): Tensor[T] = {

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/LeakyReLU.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor._
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -36,7 +36,7 @@ import scala.reflect.ClassTag
 class LeakyReLU[T: ClassTag](
   private val negval: Double = 0.01,
   var inplace: Boolean = false)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape {
   import LeakyReLU._
 
   if (negval < 0) {

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Masking.scala

@@ -16,7 +16,7 @@
 
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.{DenseTensorApply, Tensor, TensorFunc6}
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -28,7 +28,7 @@ import scala.reflect.ClassTag
  * @param maskValue mask value
  */
 class Masking[T: ClassTag](maskValue: Double = 0.0)
-(implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+(implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape{
   val batchDim = 1
   val timeDim = 2
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/PReLU.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{Initializable, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn.{Initializable, TensorModule, IdentityOutputShape}
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 import com.intel.analytics.bigdl.tensor.{DenseTensorApply, Tensor, TensorFunc4, TensorFunc6}
 import com.intel.analytics.bigdl.utils.{Engine, T, Table}
@@ -39,7 +39,8 @@ import scala.reflect.ClassTag
 @SerialVersionUID(- 877259619727212424L)
 class PReLU[T: ClassTag](
   val nOutputPlane: Int = 0)
-  (implicit ev: TensorNumeric[T]) extends TensorModule[T] with Initializable {
+  (implicit ev: TensorNumeric[T]) extends TensorModule[T]
+    with Initializable with IdentityOutputShape {
 
   val weight = if (nOutputPlane == 0) {
     Tensor[T](1)

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SReLU.scala

@@ -16,7 +16,7 @@
 
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{AbstractModule, Activity, Initializable, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn._
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 import com.intel.analytics.bigdl.utils.serializer._
@@ -48,7 +48,8 @@ import scala.reflect.ClassTag
 
 @SerialVersionUID(7173457290010080259L)
 class SReLU[T: ClassTag](val shape: Array[Int], val sharedAxes: Array[Int] = null)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] with Initializable {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T]
+    with Initializable with IdentityOutputShape {
   import SReLU._
   val weightsLen = 4
   val weights: Array[Tensor[T]] = Array.fill[Tensor[T]](4)(Tensor[T]())

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialDropout1D.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -37,7 +37,7 @@ import scala.reflect.ClassTag
 @SerialVersionUID(- 4636332259181125718L)
 class SpatialDropout1D[T: ClassTag](
   val initP: Double = 0.5)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape {
   var p = initP
   var noise = Tensor[T]()
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialDropout2D.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -41,7 +41,7 @@ import scala.reflect.ClassTag
 class SpatialDropout2D[T: ClassTag](
   val initP: Double = 0.5,
   val format: DataFormat = DataFormat.NCHW)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape {
   var p = initP
   var noise = Tensor[T]()
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/SpatialDropout3D.scala

@@ -15,7 +15,7 @@
  */
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, TensorModule}
+import com.intel.analytics.bigdl.nn.abstractnn.{DataFormat, IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 
@@ -41,7 +41,7 @@ import scala.reflect.ClassTag
 class SpatialDropout3D[T: ClassTag](
   val initP: Double = 0.5,
   val format: DataFormat = DataFormat.NCHW)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape {
   var p = initP
   var noise = Tensor[T]()
 

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/Threshold.scala

@@ -16,7 +16,7 @@
 
 package com.intel.analytics.bigdl.nn
 
-import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
+import com.intel.analytics.bigdl.nn.abstractnn.{IdentityOutputShape, TensorModule}
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 import com.intel.analytics.bigdl.tensor._
 import com.intel.analytics.bigdl.utils.Engine
@@ -37,7 +37,7 @@ import scala.reflect.ClassTag
 @SerialVersionUID(3953292249027271493L)
 class Threshold[T: ClassTag](
   private val th: Double = 1e-6, private val v: Double = 0.0, private val ip: Boolean = false)(
-  implicit ev: TensorNumeric[T]) extends TensorModule[T] {
+  implicit ev: TensorNumeric[T]) extends TensorModule[T] with IdentityOutputShape{
   var threshold = th
   var value = v
   var inPlace = ip

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/keras/ELU.scala

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2016 The BigDL Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.analytics.bigdl.nn.keras
+
+import com.intel.analytics.bigdl.nn.abstractnn._
+import com.intel.analytics.bigdl.tensor.Tensor
+import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
+import com.intel.analytics.bigdl.utils.Shape
+
+import scala.reflect.ClassTag
+
+/**
+ * Exponential Linear Unit.
+ * It follows:
+ * `f(x) =  alpha * (exp(x) - 1.) for x < 0`,
+ * `f(x) = x for x >= 0`.
+ *
+ * When you use this layer as the first layer of a model, you need to provide the argument
+ * inputShape (a Single Shape, does not include the batch dimension).
+ *
+ * @param alpha Double, scale for the negative factor.
+ * @tparam T Numeric type of parameter(e.g. weight, bias). Only support float/double now.
+ */
+class ELU[T: ClassTag](
+   val alpha: Double = 1.0,
+   val inputShape: Shape = null)(implicit ev: TensorNumeric[T])
+  extends KerasLayer[Tensor[T], Tensor[T], T](KerasLayer.addBatch(inputShape)) {
+
+  override def doBuild(inputShape: Shape): AbstractModule[Tensor[T], Tensor[T], T] = {
+    val layer = com.intel.analytics.bigdl.nn.ELU(
+      alpha = alpha,
+      inplace = false)
+    layer.asInstanceOf[AbstractModule[Tensor[T], Tensor[T], T]]
+  }
+}
+
+object ELU {
+  def apply[@specialized(Float, Double) T: ClassTag](
+    alpha: Double = 1.0,
+    inputShape: Shape = null)(implicit ev: TensorNumeric[T]): ELU[T] = {
+    new ELU[T](alpha, inputShape)
+  }
+}

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/keras/GaussianDropout.scala

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 The BigDL Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.analytics.bigdl.nn.keras
+
+import com.intel.analytics.bigdl.nn.abstractnn._
+import com.intel.analytics.bigdl.tensor.Tensor
+import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
+import com.intel.analytics.bigdl.utils.Shape
+
+import scala.reflect.ClassTag
+
+/**
+ * Apply multiplicative 1-centered Gaussian noise.
+ * As it is a regularization layer, it is only active at training time.
+ * When you use this layer as the first layer of a model, you need to provide the argument
+ * inputShape (a Single Shape, does not include the batch dimension).
+ *
+ * @param p Double, drop probability (as with `Dropout`).
+ *          The multiplicative noise will have standard deviation `sqrt(p / (1 - p))`.
+ * @tparam T Numeric type of parameter(e.g. weight, bias). Only support float/double now.
+ */
+class GaussianDropout[T: ClassTag](
+   val p: Double,
+   val inputShape: Shape = null)(implicit ev: TensorNumeric[T])
+  extends KerasLayer[Tensor[T], Tensor[T], T](KerasLayer.addBatch(inputShape)) {
+
+  override def doBuild(inputShape: Shape): AbstractModule[Tensor[T], Tensor[T], T] = {
+    val layer = com.intel.analytics.bigdl.nn.GaussianDropout(rate = p)
+    layer.asInstanceOf[AbstractModule[Tensor[T], Tensor[T], T]]
+  }
+}
+
+object GaussianDropout {
+  def apply[@specialized(Float, Double) T: ClassTag](
+    p: Double,
+    inputShape: Shape = null)(implicit ev: TensorNumeric[T]): GaussianDropout[T] = {
+    new GaussianDropout[T](p, inputShape)
+  }
+}

spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/keras/GaussianNoise.scala

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016 The BigDL Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.analytics.bigdl.nn.keras
+
+
+import com.intel.analytics.bigdl.nn.abstractnn._
+import com.intel.analytics.bigdl.tensor.Tensor
+import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
+import com.intel.analytics.bigdl.utils.Shape
+
+import scala.reflect.ClassTag
+
+/**
+ * Apply additive zero-centered Gaussian noise.
+ * This is useful to mitigate overfitting (you could see it as a form of random data augmentation).
+ * Gaussian Noise (GS) is a natural choice as corruption process for real valued inputs.
+ * As it is a regularization layer, it is only active at training time.
+ *
+ * When you use this layer as the first layer of a model, you need to provide the argument
+ * inputShape (a Single Shape, does not include the batch dimension).
+ *
+ * @param sigma Double, standard deviation of the noise distribution.
+ * @tparam T Numeric type of parameter(e.g. weight, bias). Only support float/double now.
+ */
+class GaussianNoise[T: ClassTag](
+   val sigma: Double,
+   val inputShape: Shape = null)(implicit ev: TensorNumeric[T])
+  extends KerasLayer[Tensor[T], Tensor[T], T](KerasLayer.addBatch(inputShape))  {
+
+  override def doBuild(inputShape: Shape): AbstractModule[Tensor[T], Tensor[T], T] = {
+    val layer = com.intel.analytics.bigdl.nn.GaussianNoise(stddev = sigma)
+    layer.asInstanceOf[AbstractModule[Tensor[T], Tensor[T], T]]
+  }
+}
+
+object GaussianNoise {
+  def apply[@specialized(Float, Double) T: ClassTag](
+    sigma: Double,
+    inputShape: Shape = null)(implicit ev: TensorNumeric[T]): GaussianNoise[T] = {
+    new GaussianNoise[T](sigma, inputShape)
+  }
+}