Merge pull request deeplearning4j#4029 from deeplearning4j/mp_upsampl…

…ing_layers

Upsampling 2D layers

deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java

@@ -200,6 +200,71 @@ public void testGradientCNNL1L2MLN() {
         }
     }
 
+    @Test
+    public void testCnnWithUpsampling() {
+        Nd4j.getRandom().setSeed(12345);
+        int nOut = 4;
+
+        int[] minibatchSizes = {1, 3};
+        int width = 5;
+        int height = 5;
+        int inputDepth = 1;
+
+        int[] kernel = {2, 2};
+        int[] stride = {1, 1};
+        int[] padding = {0, 0};
+        int size = 2;
+
+        String[] activations = {"sigmoid", "tanh"};
+        SubsamplingLayer.PoolingType[] poolingTypes =
+                new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
+                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};
+
+        for (String afn : activations) {
+            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
+                for (int minibatchSize : minibatchSizes) {
+                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
+                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
+                    for (int i = 0; i < minibatchSize; i++) {
+                        labels.putScalar(new int[] {i, i % nOut}, 1.0);
+                    }
+
+                    MultiLayerConfiguration conf =
+                            new NeuralNetConfiguration.Builder().regularization(false).learningRate(1.0)
+                                    .updater(Updater.SGD).weightInit(WeightInit.DISTRIBUTION)
+                                    .dist(new NormalDistribution(0, 1))
+                                    .list().layer(new ConvolutionLayer.Builder(kernel,
+                                            stride, padding).nIn(inputDepth)
+                                            .nOut(3).build())//output: (5-2+0)/1+1 = 4
+                                    .layer(new Upsampling2D.Builder().size(size).build()) //output: 4*2 =8 -> 8x8x3
+                                    .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
+                                            .activation(Activation.SOFTMAX).nIn(8 * 8 * 3)
+                                            .nOut(4).build())
+                                    .setInputType(InputType.convolutionalFlat(height, width,
+                                            inputDepth))
+                                    .build();
+
+                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
+                    net.init();
+
+                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
+                            + afn;
+
+                    if (PRINT_RESULTS) {
+                        System.out.println(msg);
+                        for (int j = 0; j < net.getnLayers(); j++)
+                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
+                    }
+
+                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
+                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
+
+                    assertTrue(msg, gradOK);
+                }
+            }
+        }
+    }
+
 
     @Test
     public void testCnnWithSubsampling() {

deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java

@@ -110,6 +110,38 @@ public void testConvnetJson() {
         assertEquals(conf, conf2);
     }
 
+    @Test
+    public void testUpsamplingConvnetJson() {
+        final int numRows = 76;
+        final int numColumns = 76;
+        int nChannels = 3;
+        int outputNum = 6;
+        int iterations = 10;
+        int seed = 123;
+
+        //setup the network
+        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations)
+                .regularization(true).l1(1e-1).l2(2e-4).useDropConnect(true).dropOut(0.5).miniBatch(true)
+                .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
+                .layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
+                        .activation(Activation.RELU).build())
+                .layer(new Upsampling2D.Builder().size(2).build())
+                .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
+                        .activation(Activation.RELU).build())
+                .layer(new Upsampling2D.Builder().size(2).build())
+                .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
+                .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
+                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
+                        .build())
+                .backprop(true).pretrain(false)
+                .setInputType(InputType.convolutional(numRows, numColumns, nChannels));
+
+        MultiLayerConfiguration conf = builder.build();
+        String json = conf.toJson();
+        MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
+        assertEquals(conf, conf2);
+    }
+
     @Test
     public void testGlobalPoolingJson() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).updater(Updater.NONE)

deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java

@@ -289,6 +289,27 @@ public void testSubSamplingWithPadding() {
         assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn());
     }
 
+    @Test
+    public void testUpsampling() {
+
+        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list()
+                .layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14
+                .layer(new Upsampling2D.Builder().size(3).build()) // 14 * 3 = 42!
+                .layer(new OutputLayer.Builder().nOut(3).build())
+                .setInputType(InputType.convolutional(28, 28, 1));
+
+        MultiLayerConfiguration conf = builder.build();
+
+        assertNotNull(conf.getInputPreProcess(2));
+        assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor);
+        CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2);
+        assertEquals(42, proc.getInputHeight());
+        assertEquals(42, proc.getInputWidth());
+        assertEquals(3, proc.getNumChannels());
+
+        assertEquals(42 * 42 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn());
+    }
+
 
     @Test
     public void testCNNDBNMultiLayer() throws Exception {

deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/Upsampling2DTest.java

@@ -0,0 +1,118 @@
+package org.deeplearning4j.nn.layers.convolution;
+
+import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator;
+import org.deeplearning4j.nn.api.Layer;
+import org.deeplearning4j.nn.conf.GradientNormalization;
+import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
+import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
+import org.deeplearning4j.nn.conf.inputs.InputType;
+import org.deeplearning4j.nn.conf.layers.OutputLayer;
+import org.deeplearning4j.nn.conf.layers.SubsamplingLayer;
+import org.deeplearning4j.nn.conf.layers.Upsampling2D;
+import org.deeplearning4j.nn.gradient.Gradient;
+import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
+import org.deeplearning4j.nn.weights.WeightInit;
+import org.junit.Test;
+import org.nd4j.linalg.activations.Activation;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.dataset.DataSet;
+import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
+import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.primitives.Pair;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Max Pumperla
+ */
+public class Upsampling2DTest {
+
+    private int nExamples = 1;
+    private int depth = 20;
+    private int nChannelsIn = 1;
+    private int inputWidth = 28;
+    private int inputHeight = 28;
+
+    private int size = 2;
+    private int outputWidth = inputWidth * size;
+    private int outputHeight = inputHeight * size;
+
+    private INDArray epsilon = Nd4j.ones(nExamples, depth, outputHeight, outputWidth);
+
+
+    @Test
+    public void testUpsampling() throws Exception {
+
+        double[] outArray = new double[] {1., 1., 2., 2., 1., 1., 2., 2., 3., 3., 4., 4., 3., 3., 4., 4.};
+        INDArray containedExpectedOut = Nd4j.create(outArray, new int[] {1, 1, 4, 4});
+        INDArray containedInput = getContainedData();
+        INDArray input = getData();
+        Layer layer = getUpsamplingLayer();
+
+        INDArray containedOutput = layer.activate(containedInput);
+        assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
+        assertEquals(containedExpectedOut, containedOutput);
+
+        INDArray output = layer.activate(input);
+        assertTrue(Arrays.equals(new int[] {nExamples, nChannelsIn, outputWidth, outputHeight},
+                        output.shape()));
+        assertEquals(nChannelsIn, output.size(1), 1e-4);
+    }
+
+
+    @Test
+    public void testUpsampling2DBackprop() throws Exception {
+        INDArray expectedContainedEpsilonInput =
+                        Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.},
+                                new int[] {1, 1, 4, 4});
+
+        INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 4., 4., 4.},
+                        new int[] {1, 1, 2, 2});
+
+        INDArray input = getContainedData();
+
+        Layer layer = getUpsamplingLayer();
+        layer.activate(input);
+
+        Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput);
+
+        assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
+        assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
+        assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);
+
+        INDArray input2 = getData();
+        layer.activate(input2);
+        int depth = input2.size(1);
+
+        epsilon = Nd4j.ones(5, depth, outputHeight, outputWidth);
+
+        Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon);
+        assertEquals(input.shape().length, out.getSecond().shape().length);
+        assertEquals(depth, out.getSecond().size(1));
+    }
+
+
+    private Layer getUpsamplingLayer() {
+        NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder()
+                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer).seed(123)
+                        .layer(new Upsampling2D.Builder(size).build()).build();
+        return conf.getLayer().instantiate(conf, null, 0, null, true);
+    }
+
+    public INDArray getData() throws Exception {
+        DataSetIterator data = new MnistDataSetIterator(5, 5);
+        DataSet mnist = data.next();
+        nExamples = mnist.numExamples();
+        return mnist.getFeatureMatrix().reshape(nExamples, nChannelsIn, inputWidth, inputHeight);
+    }
+
+    private INDArray getContainedData() {
+        INDArray ret = Nd4j.create
+                (new double[] {1., 2., 3., 4.},
+                        new int[] {1, 1, 2, 2});
+        return ret;
+    }
+
+}