a tone more tests

deeplearning4j-core/src/test/java/org/deeplearning4j/gradientcheck/CNNGradientCheckTest.java

@@ -200,6 +200,71 @@ public void testGradientCNNL1L2MLN() {
         }
     }
 
+    @Test
+    public void testCnnWithUpsampling() {
+        Nd4j.getRandom().setSeed(12345);
+        int nOut = 4;
+
+        int[] minibatchSizes = {1, 3};
+        int width = 5;
+        int height = 5;
+        int inputDepth = 1;
+
+        int[] kernel = {2, 2};
+        int[] stride = {1, 1};
+        int[] padding = {0, 0};
+        int size = 2;
+
+        String[] activations = {"sigmoid", "tanh"};
+        SubsamplingLayer.PoolingType[] poolingTypes =
+                new SubsamplingLayer.PoolingType[] {SubsamplingLayer.PoolingType.MAX,
+                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};
+
+        for (String afn : activations) {
+            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
+                for (int minibatchSize : minibatchSizes) {
+                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
+                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
+                    for (int i = 0; i < minibatchSize; i++) {
+                        labels.putScalar(new int[] {i, i % nOut}, 1.0);
+                    }
+
+                    MultiLayerConfiguration conf =
+                            new NeuralNetConfiguration.Builder().regularization(false).learningRate(1.0)
+                                    .updater(Updater.SGD).weightInit(WeightInit.DISTRIBUTION)
+                                    .dist(new NormalDistribution(0, 1))
+                                    .list().layer(new ConvolutionLayer.Builder(kernel,
+                                            stride, padding).nIn(inputDepth)
+                                            .nOut(3).build())//output: (5-2+0)/1+1 = 4
+                                    .layer(new Upsampling2D.Builder().size(size).build()) //output: 4*2 =8 -> 8x8x3
+                                    .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
+                                            .activation(Activation.SOFTMAX).nIn(8 * 8 * 3)
+                                            .nOut(4).build())
+                                    .setInputType(InputType.convolutionalFlat(height, width,
+                                            inputDepth))
+                                    .build();
+
+                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
+                    net.init();
+
+                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
+                            + afn;
+
+                    if (PRINT_RESULTS) {
+                        System.out.println(msg);
+                        for (int j = 0; j < net.getnLayers(); j++)
+                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
+                    }
+
+                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
+                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);
+
+                    assertTrue(msg, gradOK);
+                }
+            }
+        }
+    }
+
 
     @Test
     public void testCnnWithSubsampling() {

deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/MultiLayerNeuralNetConfigurationTest.java

@@ -110,6 +110,38 @@ public void testConvnetJson() {
         assertEquals(conf, conf2);
     }
 
+    @Test
+    public void testUpsamplingConvnetJson() {
+        final int numRows = 76;
+        final int numColumns = 76;
+        int nChannels = 3;
+        int outputNum = 6;
+        int iterations = 10;
+        int seed = 123;
+
+        //setup the network
+        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations)
+                .regularization(true).l1(1e-1).l2(2e-4).useDropConnect(true).dropOut(0.5).miniBatch(true)
+                .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
+                .layer(new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
+                        .activation(Activation.RELU).build())
+                .layer(new Upsampling2D.Builder().size(2).build())
+                .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
+                        .activation(Activation.RELU).build())
+                .layer(new Upsampling2D.Builder().size(2).build())
+                .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
+                .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
+                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
+                        .build())
+                .backprop(true).pretrain(false)
+                .setInputType(InputType.convolutional(numRows, numColumns, nChannels));
+
+        MultiLayerConfiguration conf = builder.build();
+        String json = conf.toJson();
+        MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
+        assertEquals(conf, conf2);
+    }
+
     @Test
     public void testGlobalPoolingJson() {
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().regularization(false).updater(Updater.NONE)

deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/ConvolutionLayerSetupTest.java

@@ -289,6 +289,27 @@ public void testSubSamplingWithPadding() {
         assertEquals(8 * 8 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn());
     }
 
+    @Test
+    public void testUpsampling() {
+
+        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().list()
+                .layer(new ConvolutionLayer.Builder(2, 2).padding(0, 0).stride(2, 2).nIn(1).nOut(3).build()) //(28-2+0)/2+1 = 14
+                .layer(new Upsampling2D.Builder().size(3).build()) // 14 * 3 = 42!
+                .layer(new OutputLayer.Builder().nOut(3).build())
+                .setInputType(InputType.convolutional(28, 28, 1));
+
+        MultiLayerConfiguration conf = builder.build();
+
+        assertNotNull(conf.getInputPreProcess(2));
+        assertTrue(conf.getInputPreProcess(2) instanceof CnnToFeedForwardPreProcessor);
+        CnnToFeedForwardPreProcessor proc = (CnnToFeedForwardPreProcessor) conf.getInputPreProcess(2);
+        assertEquals(42, proc.getInputHeight());
+        assertEquals(42, proc.getInputWidth());
+        assertEquals(3, proc.getNumChannels());
+
+        assertEquals(42 * 42 * 3, ((FeedForwardLayer) conf.getConf(2).getLayer()).getNIn());
+    }
+
 
     @Test
     public void testCNNDBNMultiLayer() throws Exception {

deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/conf/layers/Upsampling2D.java

@@ -135,21 +135,19 @@ public LayerMemoryReport getMemoryReport(InputType inputType) {
         InputType.InputTypeConvolutional c = (InputType.InputTypeConvolutional) inputType;
         InputType.InputTypeConvolutional outputType = (InputType.InputTypeConvolutional) getOutputType(-1, inputType);
 
-        // TODO: what to put here?
-//        //During forward pass: im2col array + reduce. Reduce is counted as activations, so only im2col is working mem
-//        int im2colSizePerEx =
-//                        c.getDepth() * outputType.getHeight() * outputType.getWidth() * size[0] * size[1];
-//
-//        //Current implementation does NOT cache im2col etc... which means: it's recalculated on each backward pass
-//        int trainingWorkingSizePerEx = im2colSizePerEx;
-//        if (getDropOut() > 0) {
-//            //Dup on the input before dropout, but only for training
-//            trainingWorkingSizePerEx += inputType.arrayElementsPerExample();
-//        }
+        // During forward pass: im2col array + reduce. Reduce is counted as activations, so only im2col is working mem
+        int im2colSizePerEx = c.getDepth() * outputType.getHeight() * outputType.getWidth() * size;
+
+        // Current implementation does NOT cache im2col etc... which means: it's recalculated on each backward pass
+        int trainingWorkingSizePerEx = im2colSizePerEx;
+        if (getDropOut() > 0) {
+            //Dup on the input before dropout, but only for training
+            trainingWorkingSizePerEx += inputType.arrayElementsPerExample();
+        }
 
         return new LayerMemoryReport.Builder(layerName, Upsampling2D.class, inputType, outputType)
                         .standardMemory(0, 0) //No params
-//                        .workingMemory(0, im2colSizePerEx, 0, trainingWorkingSizePerEx)
+                        .workingMemory(0, im2colSizePerEx, 0, trainingWorkingSizePerEx)
                         .cacheMemory(MemoryReport.CACHE_MODE_ALL_ZEROS, MemoryReport.CACHE_MODE_ALL_ZEROS) //No caching
                         .build();
     }

deeplearning4j-nn/src/main/java/org/deeplearning4j/nn/layers/convolution/upsampling/Upsampling2D.java

@@ -153,6 +153,10 @@ public INDArray preOutput(boolean training, boolean forBackprop) {
     @Override
     public INDArray activate(boolean training) {
 
+        if (training && conf.getLayer().getDropOut() > 0) {
+            Dropout.applyDropout(input, conf.getLayer().getDropOut());
+        }
+
         if (cacheMode == null)
             cacheMode = CacheMode.NONE;