diff --git a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java index 4d15b430eda2..f3dffac8bf11 100755 --- a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java +++ b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java @@ -792,6 +792,7 @@ private void learningRateValidation(String layerName){ private void generalValidation(String layerName){ if (useDropConnect && (Double.isNaN(dropOut) && (Double.isNaN(layer.getDropOut())))) throw new IllegalStateException(layerName +" dropConnect is set to true but dropout rate has not been added to configuration."); + if(useDropConnect && dropOut == 0.0) throw new IllegalStateException(layerName + " dropConnect is set to true but dropout rate is set to 0.0"); if (useRegularization && (Double.isNaN(l1) && layer != null && Double.isNaN(layer.getL1()) && Double.isNaN(l2) && Double.isNaN(layer.getL2()))) log.warn(layerName +" regularization is set to true but l1 or l2 has not been added to configuration."); // CompGraph may have null layers TODO confirm valid configuration @@ -801,8 +802,8 @@ private void generalValidation(String layerName){ layer.setL1(l1); if (!Double.isNaN(l2) && Double.isNaN(layer.getL2())) layer.setL2(l2); - } else if (!Double.isNaN(l1) || !Double.isNaN(layer.getL1()) || !Double.isNaN(l2) || !Double.isNaN(layer.getL2())) - log.warn(layerName +" l1 or l2 has been added to configuration but useRegularization is set to false."); + } else if (!useRegularization && (!Double.isNaN(l1) || !Double.isNaN(layer.getL1()) || !Double.isNaN(l2) || !Double.isNaN(layer.getL2())) ) + throw new IllegalStateException(layerName +" l1 or l2 has been added to configuration but useRegularization is set to false."); if (Double.isNaN(l2) && Double.isNaN(layer.getL2())) layer.setL2(0.0); if (Double.isNaN(l1) && Double.isNaN(layer.getL1())) diff --git a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java index 0b4ccecaa777..fe45e075f37b 100644 --- a/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java +++ b/deeplearning4j-core/src/main/java/org/deeplearning4j/nn/updater/BaseUpdater.java @@ -100,7 +100,7 @@ public void applyLrDecayPolicy(LearningRatePolicy decay, Layer layer, int iterat conf.setLearningRateByParam(variable, lr * Math.pow(decayRate, Math.floor(iteration/conf.getLrPolicySteps()))); break; case Poly: - conf.setLearningRateByParam(variable, lr * Math.pow((1 - (iteration * 1.0)/conf.getNumIterations()), conf.getLrPolicyPower())); + conf.setLearningRateByParam(variable, lr * Math.pow((1 - ((double)iteration)/conf.getNumIterations()), conf.getLrPolicyPower())); break; case Sigmoid: conf.setLearningRateByParam(variable, lr / (1 + Math.exp(-decayRate * (iteration - conf.getLrPolicySteps())))); diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java index fe947ca445d1..e9ec7650ccde 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/datasets/iterator/DataSetIteratorTest.java @@ -26,6 +26,7 @@ import org.deeplearning4j.nn.weights.WeightInit; import org.deeplearning4j.optimize.api.IterationListener; import org.deeplearning4j.optimize.listeners.ScoreIterationListener; +import org.junit.Ignore; import org.junit.Test; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.dataset.DataSet; @@ -163,6 +164,7 @@ public void testLfwModel() throws Exception{ } @Test + @Ignore //Until CifarDataSetIterator / CifarLoader is fixed public void testCifarIterator() throws Exception { int numExamples = 10; int row = 28; @@ -177,6 +179,7 @@ public void testCifarIterator() throws Exception { @Test + @Ignore //Until CifarDataSetIterator / CifarLoader is fixed public void testCifarModel() throws Exception{ final int height = 32; final int width = 32; diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java index d745c27f3b24..fb251508c388 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/NeuralNetConfigurationTest.java @@ -250,6 +250,7 @@ public void testLearningRateByParam(){ INDArray gradientW = Nd4j.ones(nIns[0], nOuts[0]); MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() + .learningRate(0.3) .list() .layer(0, new DenseLayer.Builder().nIn(nIns[0]).nOut(nOuts[0]).updater(org.deeplearning4j.nn.conf.Updater.SGD).learningRate(lr).biasLearningRate(biasLr).build()) .layer(1, new BatchNormalization.Builder().nIn(nIns[1]).nOut(nOuts[1]).learningRate(0.7).build()) @@ -265,7 +266,9 @@ public void testLearningRateByParam(){ assertEquals(lr, net.getLayer(0).conf().getLearningRateByParam("W"), 1e-4); assertEquals(biasLr, net.getLayer(0).conf().getLearningRateByParam("b"), 1e-4); assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("gamma"), 1e-4); - assertEquals(0.1, net.getLayer(1).conf().getLearningRateByParam("b"), 1e-4); + assertEquals(0.7, net.getLayer(1).conf().getLearningRateByParam("b"), 1e-4); //If not explicitly set, bias learning rate should be same as weights LR + assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("W"), 1e-4); //From global LR + assertEquals(0.3, net.getLayer(2).conf().getLearningRateByParam("b"), 1e-4); //From global LR } @Test diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java index 9ed0bd3a6590..a4c56260aff3 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/conf/layers/LayerConfigValidationTest.java @@ -185,7 +185,7 @@ public void testLRPolicyMissingDecayRate(){ MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() .iterations(iterations) .learningRate(lr) - .learningRateDecayPolicy(LearningRatePolicy.Poly) + .learningRateDecayPolicy(LearningRatePolicy.Inverse) .lrPolicyPower(power) .list() .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build() ) @@ -312,6 +312,7 @@ public void testPredefinedConfigValues() { conf = new NeuralNetConfiguration.Builder() .learningRate(0.3) .updater(Updater.ADAM) + .regularization(true) .weightInit(WeightInit.DISTRIBUTION) .list() .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).l2(0.5).l1(0.3).build()) diff --git a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java index 6ae30bd4ec72..8af16d1a8c4c 100644 --- a/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java +++ b/deeplearning4j-core/src/test/java/org/deeplearning4j/nn/layers/convolution/SubsamplingLayerTest.java @@ -135,6 +135,7 @@ public void testSubSampleLayerMaxBackprop() throws Exception { INDArray input2 = getData(); layer.activate(input2); + int depth = input2.size(1); epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);