Channel-Wise RReLU

Author: jasonkuen

lib/THNN/generic/RReLU.c

@@ -17,7 +17,7 @@ void THNN_(RReLU_updateOutput)(
   if (channelwise && train)
   {
     long bs, ks;
-    THIndex_t nOutputPlane;
+    long nOutputPlane;
     {
       long input_ndim = THTensor_(nDimension)(input);
       switch (input_ndim)
@@ -39,7 +39,7 @@ void THNN_(RReLU_updateOutput)(
           ks = input->size[2] * input->size[3];
           break;
       }
-      nOutputPlane = input->size[(input_ndim + 1) % 2]
+      nOutputPlane = input->size[(input_ndim + 1) % 2];
     }
     // get default random generator
     if (inplace)
@@ -51,14 +51,17 @@ void THNN_(RReLU_updateOutput)(
     real *input_data = THTensor_(data)(input);
     real *noise_data = THTensor_(data)(noise);
     if (!inplace)
+    {
+      THTensor_(resizeAs)(output, input);
       output_data = THTensor_(data)(output);
+    }
     THTensor *channel_noise = THTensor_(newWithSize1d)(nOutputPlane);
     real *channel_noise_data = THTensor_(data)(channel_noise);
 
     THIndex_t i, j, k;
 #pragma omp parallel for private(j)
     for (j = 0; j < nOutputPlane; ++j)
-      channel_noise_data[j] = (real)THRandom_uniform(generator, lower, upper)
+      channel_noise_data[j] = (real)THRandom_uniform(generator, lower, upper);
 #pragma omp parallel for private(j,k)
     for (i = 0; i < bs; ++i)
     {
@@ -72,17 +75,22 @@ void THNN_(RReLU_updateOutput)(
       for (j = 0; j < nOutputPlane; ++j)
       {
         const real r = channel_noise_data[j];
-          for (k = 0; k < ks; ++k)
-            if (inplace)
-              if n_input_data[k] <= 0
-              {
-                n_input_data[k] = r * n_input_data[k];
-                n_noise_data[k] = r;
-              }
-              else
-                n_noise_data[k] = 1;
+        for (k = 0; k < ks; ++k)
+          if (inplace)
+            if (n_input_data[k] <= 0)
+            {
+              n_input_data[k] = r * n_input_data[k];
+              n_noise_data[k] = r;
+            }
             else
-              n_output_data[k] = (n_input_data[k] > 0) ? n_input_data[k] : r * n_input_data[k];
+              n_noise_data[k] = 1;
+          else
+            n_output_data[k] = (n_input_data[k] > 0) ? n_input_data[k] : r * n_input_data[k];
+        n_input_data += ks;
+        if (inplace)
+          n_noise_data += ks;
+        else
+          n_output_data += ks;
       }
     }
     if (inplace)
@@ -172,7 +180,7 @@ void THNN_(RReLU_updateGradInput)(
     if (channelwise && !inplace)
     {
       long bs, ks;
-      THIndex_t nOutputPlane;
+      long nOutputPlane;
       {
         long input_ndim = THTensor_(nDimension)(input);
         switch (input_ndim)
@@ -194,12 +202,12 @@ void THNN_(RReLU_updateGradInput)(
             ks = input->size[2] * input->size[3];
             break;
         }
-        nOutputPlane = input->size[(input_ndim + 1) % 2]
+        nOutputPlane = input->size[(input_ndim + 1) % 2];
       }
 
-      const real *output_data = output_data = THTensor_(data)(output);
       const real *input_data = THTensor_(data)(input);
       const real *gradOutput_data = THTensor_(data)(gradOutput);
+      THTensor_(resizeAs)(gradInput, input);
       real *gradInput_data = THTensor_(data)(gradInput);
       const real *noise_data = THTensor_(data)(noise);
 
@@ -215,12 +223,10 @@ void THNN_(RReLU_updateGradInput)(
         {
           const real r = noise_data[j];
           for (k = 0; k < ks; ++k)
-          {
             if (n_input_data[k] > 0)
               n_gradInput_data[k] = n_gradOutput_data[k];
             else
               n_gradInput_data[k] = n_gradOutput_data[k] * r;
-          }
           n_input_data += ks;
           n_gradInput_data += ks;
           n_gradOutput_data += ks;

test.lua

@@ -484,32 +484,35 @@ function nntest.RReLU()
    for _,train in ipairs({true,false}) do
       -- test with separate output buffer and inplace
       for _,inplace in ipairs({false,true}) do
-         module = nn.RReLU(l, u, inplace)
-         if train then
-            module:training()
-         else
-            module:evaluate()
-         end
-         input = torch.rand(nframe, size, kW, kH) - 0.5
-         input:storage()[1] = -1
-         local original_input = input:clone()
-         local output = module:forward(input)
-         mytester:assert(output:sign():eq(original_input:sign()):all(), 'sign flipped forward ')
-         local gradOutput = torch.ones(output:size())
-         local gradInput = module:backward(input, gradOutput)
-         mytester:assert(gradInput:gt(0):eq(input:ne(0)):all(), 'gradient ')
-         mytester:assert(gradInput:lt(1):eq(input:le(0)):all(), 'backward negative inputs ')
-         mytester:assert(gradInput:eq(1):eq(input:gt(0)):all(), 'backward positive inputs ')
-         if not train then
-            local err = gradInput[input:le(0)]:mean()-(module.lower+module.upper)/2
-            mytester:assertlt(err, precision, 'error on gradient ')
-         end
+          -- test with channel-wise
+          for _,cw in ipairs({true,false}) do
+            module = nn.RReLU(l, u, inplace, cw)
+            if train then
+              module:training()
+            else
+              module:evaluate()
+            end
+            input = torch.rand(nframe, size, kW, kH) - 0.5
+            input:storage()[1] = -1
+            local original_input = input:clone()
+            local output = module:forward(input)
+            mytester:assert(output:sign():eq(original_input:sign()):all(), 'sign flipped forward ')
+            local gradOutput = torch.ones(output:size())
+            local gradInput = module:backward(input, gradOutput)
+            mytester:assert(gradInput:gt(0):eq(input:ne(0)):all(), 'gradient ')
+            mytester:assert(gradInput:lt(1):eq(input:le(0)):all(), 'backward negative inputs ')
+            mytester:assert(gradInput:eq(1):eq(input:gt(0)):all(), 'backward positive inputs ')
+            if not train then
+              local err = gradInput[input:le(0)]:mean()-(module.lower+module.upper)/2
+              mytester:assertlt(err, precision, 'error on gradient ')
+            end
 
-         input = -torch.rand(1000)
-         module:forward(input) -- fill internal noise tensor
-         local g = module:backward(input, torch.ones(1000))
-         local err = math.abs(g[input:le(0)]:mean()-(module.lower+module.upper)/2)
-         mytester:assertlt(err, 0.05, 'mean deviation of gradient for negative inputs ')
+            input = -torch.rand(1000)
+            module:forward(input) -- fill internal noise tensor
+            local g = module:backward(input, torch.ones(1000))
+            local err = math.abs(g[input:le(0)]:mean()-(module.lower+module.upper)/2)
+            mytester:assertlt(err, 0.05, 'mean deviation of gradient for negative inputs ')
+        end
       end
    end
 end