Fix input validation, LR by param and subsampling backprop tests

vovietanh · May 2, 2016 · dcb0404 · dcb0404
1 parent 658a6db
commit dcb0404
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 5 deletions.
diff --git a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java
@@ -792,6 +792,7 @@ private void learningRateValidation(String layerName){
         private void generalValidation(String layerName){
             if (useDropConnect && (Double.isNaN(dropOut) && (Double.isNaN(layer.getDropOut()))))
                 throw new IllegalStateException(layerName +" dropConnect is set to true but dropout rate has not been added to configuration.");
+            if(useDropConnect && dropOut == 0.0) throw new IllegalStateException(layerName + " dropConnect is set to true but dropout rate is set to 0.0");
             if (useRegularization && (Double.isNaN(l1) && layer != null && Double.isNaN(layer.getL1()) && Double.isNaN(l2) && Double.isNaN(layer.getL2())))
                 log.warn(layerName +" regularization is set to true but l1 or l2 has not been added to configuration.");
             // CompGraph may have null layers TODO confirm valid configuration
@@ -801,8 +802,8 @@ private void generalValidation(String layerName){
                         layer.setL1(l1);
                     if (!Double.isNaN(l2) && Double.isNaN(layer.getL2()))
                         layer.setL2(l2);
-                } else if (!Double.isNaN(l1) || !Double.isNaN(layer.getL1()) || !Double.isNaN(l2) || !Double.isNaN(layer.getL2()))
-                    log.warn(layerName +" l1 or l2 has been added to configuration but useRegularization is set to false.");
+                } else if (!useRegularization && (!Double.isNaN(l1) || !Double.isNaN(layer.getL1()) || !Double.isNaN(l2) || !Double.isNaN(layer.getL2())) )
+                    throw new IllegalStateException(layerName +" l1 or l2 has been added to configuration but useRegularization is set to false.");
                 if (Double.isNaN(l2) && Double.isNaN(layer.getL2()))
                     layer.setL2(0.0);
                 if (Double.isNaN(l1) && Double.isNaN(layer.getL1()))

diff --git a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java
@@ -100,7 +100,7 @@ public void applyLrDecayPolicy(LearningRatePolicy decay, Layer layer, int iterat
                 conf.setLearningRateByParam(variable, lr * Math.pow(decayRate, Math.floor(iteration/conf.getLrPolicySteps())));
                 break;
             case Poly:
-                conf.setLearningRateByParam(variable, lr * Math.pow((1 - (iteration * 1.0)/conf.getNumIterations()), conf.getLrPolicyPower()));
+                conf.setLearningRateByParam(variable, lr * Math.pow((1 - ((double)iteration)/conf.getNumIterations()), conf.getLrPolicyPower()));
                 break;
             case Sigmoid:
                 conf.setLearningRateByParam(variable, lr / (1 + Math.exp(-decayRate * (iteration - conf.getLrPolicySteps()))));

diff --git a/...arning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java b/...arning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java
@@ -26,6 +26,7 @@
 import org.deeplearning4j.nn.weights.WeightInit;
 import org.deeplearning4j.optimize.api.IterationListener;
 import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
@@ -163,6 +164,7 @@ public void testLfwModel() throws Exception{
 	}
 
 	@Test
+	@Ignore	//Until CifarDataSetIterator / CifarLoader is fixed
 	public void testCifarIterator() throws Exception {
 		int numExamples = 10;
 		int row = 28;
@@ -177,6 +179,7 @@ public void testCifarIterator() throws Exception {
 
 
 	@Test
+	@Ignore	//Until CifarDataSetIterator / CifarLoader is fixed
 	public void testCifarModel() throws Exception{
 		final int height = 32;
 		final int width = 32;

diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java
@@ -250,6 +250,7 @@ public void testLearningRateByParam(){
         INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]);
 
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
+                .learningRate(0.3)
                 .list()
                 .layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).learningRate(lr).biasLearningRate(biasLr).build())
                 .layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).learningRate(0.7).build())
@@ -265,7 +266,9 @@ public void testLearningRateByParam(){
         assertEquals(lr, net.getLayer(0).conf().getLearningRateByParam("W"), 1e-4);
         assertEquals(biasLr, net.getLayer(0).conf().getLearningRateByParam("b"), 1e-4);
         assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("gamma"), 1e-4);
-        assertEquals(0.1, net.getLayer(1).conf().getLearningRateByParam("b"), 1e-4);
+        assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("b"), 1e-4);        //If not explicitly set, bias learning rate should be same as weights LR
+        assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("W"), 1e-4);        //From global LR
+        assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("b"), 1e-4);        //From global LR
     }
 
     @Test

diff --git a/...ing4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java b/...ing4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java
@@ -185,7 +185,7 @@ public void testLRPolicyMissingDecayRate(){
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                 .iterations(iterations)
                 .learningRate(lr)
-                .learningRateDecayPolicy(LearningRatePolicy.Poly)
+                .learningRateDecayPolicy(LearningRatePolicy.Inverse)
                 .lrPolicyPower(power)
                 .list()
                 .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build() )
@@ -312,6 +312,7 @@ public void testPredefinedConfigValues() {
         conf = new NeuralNetConfiguration.Builder()
                 .learningRate(0.3)
                 .updater(Updater.ADAM)
+                .regularization(true)
                 .weightInit(WeightInit.DISTRIBUTION)
                 .list()
                 .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).l1(0.3).build())

diff --git a/...g4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java b/...g4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java
@@ -135,6 +135,7 @@ public void testSubSampleLayerMaxBackprop() throws Exception {
 
         INDArray input2 = getData();
         layer.activate(input2);
+        int depth = input2.size(1);
 
         epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);