Use custom range in HardTanh and mask it as Clamp

allows HardTanh to use a custom interval for linear region when min/max values
are specified in declaration. Thus its implementation can be shared with new
Clamp function.

Author: jhjin

Clamp.lua

@@ -0,0 +1,5 @@
+local Clamp, Parent = torch.class('nn.Clamp', 'nn.HardTanh')
+
+function Clamp:__init(min_value, max_value)
+   Parent.__init(self, min_value, max_value)
+end

HardTanh.lua

@@ -1,6 +1,15 @@
-local HardTanh = torch.class('nn.HardTanh', 'nn.Module')
+local HardTanh, parent = torch.class('nn.HardTanh', 'nn.Module')
+
+function HardTanh:__init(min_value, max_value)
+   parent.__init(self)
+   self.min_val = min_value or -1
+   self.max_val = max_value or 1
+   assert(self.max_val>self.min_val, 'max_value must be larger than min_value')
+end
 
 function HardTanh:updateOutput(input)
+   self.min_val = self.min_val or -1
+   self.max_val = self.max_val or 1
    return input.nn.HardTanh_updateOutput(self, input)
 end
 

doc/simple.md

@@ -29,6 +29,7 @@ Simple Modules are used for various tasks like adapting Tensor methods and provi
     * [Power](#nn.Power) : an element-wise [pow](https://github.com/torch/torch7/blob/master/doc/maths.md#res-torchpowres-x) operation ;
     * [Square](#nn.Square) : an element-wise square operation ;
     * [Sqrt](#nn.Sqrt) : an element-wise [sqrt](https://github.com/torch/torch7/blob/master/doc/maths.md#res-torchsqrtres-x) operation ;
+    * [Clamp](#nn.Clamp) : an element-wise [clamp](https://github.com/torch/torch7/blob/master/doc/maths.md#res-torchclampres-tensor1-min_value-max_value) operation ;
     * [Normalize](#nn.Normalize) : normalizes the input to have unit `L_p` norm ;
     * [MM](#nn.MM) : matrix-matrix multiplication (also supports batches of matrices) ;
   * Miscellaneous Modules :
@@ -905,6 +906,32 @@ gnuplot.grid(true)
 
 ![](image/power.png)
 
+<a name="nn.Clamp"></a>
+## Clamp ##
+
+```lua
+module = nn.Clamp(min_value, max_value)
+```
+
+Clamps all elements into the range `[min_value, max_value]`.
+Output is identical to input in the range, otherwise elements less than `min_value` (or greater than `max_value`) are saturated to `min_value` (or `max_value`).
+
+```lua
+A = torch.randn(2, 5)
+m = nn.Clamp(-0.1, 0.5)
+B = m:forward(A)
+
+print(A)  -- input
+-1.1321  0.0227 -0.4672  0.6519 -0.5380
+ 0.9061 -1.0858  0.3697 -0.8120 -1.6759
+[torch.DoubleTensor of size 3x5]
+
+print(B)  -- output
+-0.1000  0.0227 -0.1000  0.5000 -0.1000
+ 0.5000 -0.1000  0.3697 -0.1000 -0.1000
+[torch.DoubleTensor of size 3x5]
+```
+
 <a name="nn.Normalize"></a>
 ## Normalize ##
 

doc/transfer.md

@@ -14,6 +14,10 @@ thus outputting a Tensor of the same dimension.
   * `f(x)` = `-1, if x <`  `-1,`
   * `f(x)` = `x,` `otherwise.`
 
+The range of the linear region `[-1 1]` can be adjusted by specifying arguments in declaration, for example `nn.HardTanh(min_value, max_value)`.
+Otherwise, `[min_value max_value]` is set to `[-1 1]` by default.
+
+
 ```lua
 ii=torch.linspace(-2,2)
 m=nn.HardTanh()

generic/HardTanh.c

@@ -5,19 +5,21 @@
 static int nn_(HardTanh_updateOutput)(lua_State *L)
 {
   THTensor *input = luaT_checkudata(L, 2, torch_Tensor);
+  real min_val = luaT_getfieldchecknumber(L, 1, "min_val");
+  real max_val = luaT_getfieldchecknumber(L, 1, "max_val");
   THTensor *output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor);
 
   THTensor_(resizeAs)(output, input);
 
   if (input->nDimension == 1 || !THTensor_(isContiguous)(input) || !THTensor_(isContiguous)(output))
   {
     TH_TENSOR_APPLY2(real, output, real, input,     \
-         if(*input_data < -1)     \
-           *output_data = -1;   \
-         else if(*input_data <= 1)    \
+         if(*input_data < min_val)     \
+           *output_data = min_val;   \
+         else if(*input_data <= max_val)    \
            *output_data = *input_data;  \
          else       \
-           *output_data = 1;);
+           *output_data = max_val;);
   }
   else
   {
@@ -28,12 +30,12 @@ static int nn_(HardTanh_updateOutput)(lua_State *L)
 #pragma omp parallel for private(i)
     for (i = 0; i < THTensor_(nElement)(input); i++)
     {
-      if(ptr_input[i] < -1)
-	ptr_output[i] = -1;
-      else if (ptr_input[i] <= 1)
-	ptr_output[i] = ptr_input[i];
+      if(ptr_input[i] < min_val)
+        ptr_output[i] = min_val;
+      else if (ptr_input[i] <= max_val)
+        ptr_output[i] = ptr_input[i];
       else
-	ptr_output[i] = 1;
+        ptr_output[i] = max_val;
     }
   }
   return 1;
@@ -42,6 +44,8 @@ static int nn_(HardTanh_updateOutput)(lua_State *L)
 static int nn_(HardTanh_updateGradInput)(lua_State *L)
 {
   THTensor *input = luaT_checkudata(L, 2, torch_Tensor);
+  real min_val = luaT_getfieldchecknumber(L, 1, "min_val");
+  real max_val = luaT_getfieldchecknumber(L, 1, "max_val");
   THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor);
   THTensor *gradInput = luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor);
 
@@ -53,7 +57,7 @@ static int nn_(HardTanh_updateGradInput)(lua_State *L)
       !THTensor_(isContiguous)(gradInput))
   {
     TH_TENSOR_APPLY3(real, gradInput, real, gradOutput, real, input,  \
-         if(*input_data < -1 || *input_data > 1)    \
+         if(*input_data < min_val || *input_data > max_val)    \
            *gradInput_data = 0;                             \
          else           \
            *gradInput_data = *gradOutput_data;);
@@ -68,10 +72,10 @@ static int nn_(HardTanh_updateGradInput)(lua_State *L)
 #pragma omp parallel for private(i)
     for (i = 0; i < THTensor_(nElement)(input); i++)
     {
-      if(ptr_input[i] < -1 || ptr_input[i] > 1)
-	ptr_gradInput[i] = 0;
+      if(ptr_input[i] < min_val || ptr_input[i] > max_val)
+        ptr_gradInput[i] = 0;
       else
-	ptr_gradInput[i] = ptr_gradOutput[i];
+        ptr_gradInput[i] = ptr_gradOutput[i];
     }
   }
   return 1;

init.lua

@@ -52,6 +52,7 @@ include('Normalize.lua')
 include('Exp.lua')
 include('Log.lua')
 include('HardTanh.lua')
+include('Clamp.lua')
 include('LogSigmoid.lua')
 include('LogSoftMax.lua')
 include('Sigmoid.lua')

test.lua

@@ -266,6 +266,24 @@ function nntest.HardTanh()
    mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ')
 end
 
+function nntest.Clamp()
+   local ini = math.random(3,5)
+   local inj = math.random(3,5)
+   local ink = math.random(3,5)
+   local max_value =  math.abs(math.random())
+   local min_value = -math.abs(math.random())
+   local input = torch.Tensor(ink, inj, ini):zero()
+
+   local module = nn.Clamp(min_value, max_value)
+
+   local err = jac.testJacobian(module, input)
+   mytester:assertlt(err, precision ,  'error on state ')
+
+   local ferr, berr = jac.testIO(module, input)
+   mytester:asserteq(ferr, 0, torch.typename(module) .. ' - i/o forward err ')
+   mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ')
+end
+
 function nntest.Abs()
    local ini = math.random(3,5)
    local inj = math.random(3,5)