pick up distributions in weight-bearing layers

KraFusion · Sep 13, 2017 · 5c084b5 · 5c084b5
1 parent c721035
commit 5c084b5
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 22 deletions.
diff --git a/...rg/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java b/...rg/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution1D.java
@@ -18,11 +18,14 @@
 package org.deeplearning4j.nn.modelimport.keras.layers.convolutional;
 
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.Convolution1DLayer;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
+import org.deeplearning4j.nn.weights.WeightInit;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.Map;
 
@@ -80,18 +83,24 @@ public KerasAtrousConvolution1D(Map<String, Object> layerConfig, boolean enforce
         LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         Convolution1DLayer.Builder builder = new Convolution1DLayer.Builder().name(this.layerName)
                 .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout)
                 .activation(getActivationFromConfig(layerConfig, conf))
-                .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
-                        enforceTrainingConfig, conf, kerasMajorVersion))
+                .weightInit(weightInit)
                 .dilation(getDilationRate(layerConfig, 1, conf, true))
                 .l1(this.weightL1Regularization).l2(this.weightL2Regularization)
                 .convolutionMode(getConvolutionModeFromConfig(layerConfig, conf))
                 .kernelSize(getKernelSizeFromConfig(layerConfig, 1,  conf, kerasMajorVersion)[0])
                 .hasBias(hasBias)
                 .stride(getStrideFromConfig(layerConfig, 1, conf)[0]);
         int[] padding = getPaddingFromBorderModeConfig(layerConfig, 1, conf, kerasMajorVersion);
+        if (distribution != null)
+            builder.dist(distribution);
         if (hasBias)
             builder.biasInit(0.0);
         if (padding != null)

diff --git a/...rg/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java b/...rg/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasAtrousConvolution2D.java
@@ -18,11 +18,14 @@
 package org.deeplearning4j.nn.modelimport.keras.layers.convolutional;
 
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.ConvolutionLayer;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
+import org.deeplearning4j.nn.weights.WeightInit;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.Map;
 
@@ -82,18 +85,24 @@ public KerasAtrousConvolution2D(Map<String, Object> layerConfig, boolean enforce
         LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         ConvolutionLayer.Builder builder = new ConvolutionLayer.Builder().name(this.layerName)
                 .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout)
                 .activation(getActivationFromConfig(layerConfig, conf))
-                .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
-                        enforceTrainingConfig, conf, kerasMajorVersion))
+                .weightInit(weightInit)
                 .dilation(getDilationRate(layerConfig, 2, conf, true))
                 .l1(this.weightL1Regularization).l2(this.weightL2Regularization)
                 .convolutionMode(getConvolutionModeFromConfig(layerConfig, conf))
                 .kernelSize(getKernelSizeFromConfig(layerConfig, 2, conf, kerasMajorVersion))
                 .hasBias(hasBias)
                 .stride(getStrideFromConfig(layerConfig, 2, conf));
         int[] padding = getPaddingFromBorderModeConfig(layerConfig, 2, conf, kerasMajorVersion);
+        if (distribution != null)
+            builder.dist(distribution);
         if (hasBias)
             builder.biasInit(0.0);
         if (padding != null)

diff --git a/...java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java b/...java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution1D.java
@@ -20,11 +20,14 @@
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.Convolution1DLayer;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
+import org.deeplearning4j.nn.weights.WeightInit;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.Map;
 
@@ -84,17 +87,23 @@ public KerasConvolution1D(Map<String, Object> layerConfig, boolean enforceTraini
         LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         Convolution1DLayer.Builder builder = new Convolution1DLayer.Builder().name(this.layerName)
                 .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout)
                 .activation(getActivationFromConfig(layerConfig, conf))
-                .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
-                        enforceTrainingConfig, conf, kerasMajorVersion))
+                .weightInit(weightInit)
                 .l1(this.weightL1Regularization).l2(this.weightL2Regularization)
                 .convolutionMode(getConvolutionModeFromConfig(layerConfig, conf))
                 .kernelSize(getKernelSizeFromConfig(layerConfig, 1,  conf, kerasMajorVersion)[0])
                 .hasBias(hasBias)
                 .stride(getStrideFromConfig(layerConfig, 1, conf)[0]);
         int[] padding = getPaddingFromBorderModeConfig(layerConfig, 1, conf, kerasMajorVersion);
+        if (distribution != null)
+            builder.dist(distribution);
         if (hasBias)
             builder.biasInit(0.0);
         if (padding != null)

diff --git a/...java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java b/...java/org/deeplearning4j/nn/modelimport/keras/layers/convolutional/KerasConvolution2D.java
@@ -20,11 +20,14 @@
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.ConvolutionLayer;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
+import org.deeplearning4j.nn.weights.WeightInit;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.Map;
 
@@ -82,6 +85,11 @@ public KerasConvolution2D(Map<String, Object> layerConfig, boolean enforceTraini
         numTrainableParams = hasBias ? 2 : 1;
         int[] dilationRate = getDilationRate(layerConfig, 2, conf, false);
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion);
         LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
@@ -90,14 +98,15 @@ public KerasConvolution2D(Map<String, Object> layerConfig, boolean enforceTraini
         ConvolutionLayer.Builder builder = new ConvolutionLayer.Builder().name(this.layerName)
                 .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout)
                 .activation(getActivationFromConfig(layerConfig, conf))
-                .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
-                        enforceTrainingConfig, conf, kerasMajorVersion))
+                .weightInit(weightInit)
                 .l1(this.weightL1Regularization).l2(this.weightL2Regularization)
                 .convolutionMode(getConvolutionModeFromConfig(layerConfig, conf))
                 .kernelSize(getKernelSizeFromConfig(layerConfig, 2, conf, kerasMajorVersion))
                 .hasBias(hasBias)
                 .stride(getStrideFromConfig(layerConfig, 2, conf));
         int[] padding = getPaddingFromBorderModeConfig(layerConfig, 2, conf, kerasMajorVersion);
+        if (distribution != null)
+            builder.dist(distribution);
         if (hasBias)
             builder.biasInit(0.0);
         if (padding != null)

diff --git a/...limport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java b/...limport/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/core/KerasDense.java
@@ -4,14 +4,17 @@
 import lombok.extern.slf4j.Slf4j;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.DenseLayer;
 import org.deeplearning4j.nn.modelimport.keras.KerasLayer;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
 import org.deeplearning4j.nn.params.DefaultParamInitializer;
+import org.deeplearning4j.nn.weights.WeightInit;
 import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -76,13 +79,19 @@ public KerasDense(Map<String, Object> layerConfig, boolean enforceTrainingConfig
         LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         DenseLayer.Builder builder = new DenseLayer.Builder().name(this.layerName).nOut(getNOutFromConfig(layerConfig, conf))
                 .dropOut(this.dropout).activation(getActivationFromConfig(layerConfig, conf))
-                .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
-                        enforceTrainingConfig, conf, kerasMajorVersion))
+                .weightInit(weightInit)
                 .biasInit(0.0)
                 .l1(this.weightL1Regularization).l2(this.weightL2Regularization)
                 .hasBias(hasBias);
+        if (distribution != null)
+            builder.dist(distribution);
         if (biasConstraint != null)
             builder.constrainBias(biasConstraint);
         if (weightConstraint != null)

diff --git a/...c/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java b/...c/main/java/org/deeplearning4j/nn/modelimport/keras/layers/embeddings/KerasEmbedding.java
@@ -4,6 +4,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.EmbeddingLayer;
 import org.deeplearning4j.nn.modelimport.keras.KerasLayer;
@@ -12,8 +13,10 @@
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasConstraintUtils;
 import org.deeplearning4j.nn.modelimport.keras.utils.KerasLayerUtils;
 import org.deeplearning4j.nn.params.DefaultParamInitializer;
+import org.deeplearning4j.nn.weights.WeightInit;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.ndarray.INDArray;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -70,15 +73,21 @@ public KerasEmbedding(Map<String, Object> layerConfig, boolean enforceTrainingCo
                     "in DL4J, apply masking as a pre-processing step to your input." +
                     "See https://deeplearning4j.org/usingrnns#masking for more on this.");
 
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_EMBEDDING_INIT(),
+                enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
         LayerConstraint embeddingConstraint = KerasConstraintUtils.getConstraintsFromConfig(
                 layerConfig, conf.getLAYER_FIELD_EMBEDDINGS_CONSTRAINT(), conf, kerasMajorVersion);
 
         EmbeddingLayer.Builder builder = new EmbeddingLayer.Builder().name(this.layerName).nIn(inputDim)
                         .nOut(getNOutFromConfig(layerConfig, conf)).dropOut(this.dropout).activation(Activation.IDENTITY)
-                        .weightInit(getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_EMBEDDING_INIT(),
-                                enforceTrainingConfig, conf, kerasMajorVersion))
+                        .weightInit(weightInit)
                         .biasInit(0.0)
                         .l1(this.weightL1Regularization).l2(this.weightL2Regularization).hasBias(false);
+        if (distribution != null)
+            builder.dist(distribution);
         if (embeddingConstraint != null)
             builder.constrainWeights(embeddingConstraint);
         this.layer = builder.build();

diff --git a/...ort/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLstm.java b/...ort/src/main/java/org/deeplearning4j/nn/modelimport/keras/layers/recurrent/KerasLstm.java
@@ -4,6 +4,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.deeplearning4j.nn.api.layers.LayerConstraint;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
+import org.deeplearning4j.nn.conf.distribution.Distribution;
 import org.deeplearning4j.nn.conf.inputs.InputType;
 import org.deeplearning4j.nn.conf.layers.LSTM;
 import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor;
@@ -19,6 +20,7 @@
 import org.nd4j.linalg.factory.Nd4j;
 import org.nd4j.linalg.indexing.INDArrayIndex;
 import org.nd4j.linalg.indexing.NDArrayIndex;
+import org.nd4j.linalg.primitives.Pair;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -98,17 +100,24 @@ public KerasLstm(Map<String, Object> layerConfig)
     public KerasLstm(Map<String, Object> layerConfig, boolean enforceTrainingConfig)
                     throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
         super(layerConfig, enforceTrainingConfig);
-        WeightInit weightInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
+
+        Pair<WeightInit, Distribution> init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
                 enforceTrainingConfig, conf, kerasMajorVersion);
-        WeightInit recurrentWeightInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(),
+        WeightInit weightInit = init.getFirst();
+        Distribution distribution = init.getSecond();
+
+        Pair<WeightInit, Distribution> recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(),
                 enforceTrainingConfig, conf, kerasMajorVersion);
+        WeightInit recurrentWeightInit = recurrentInit.getFirst();
+        Distribution recurrentDistribution = recurrentInit.getSecond();
+
         Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
         Boolean returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES());
         if (!returnSequences) {
             log.warn("Keras setting 'return_sequences = False' is not properly supported," +
                     "DL4J's LSTM layer returns sequences by default");
         }
-        if (weightInit != recurrentWeightInit)
+        if (weightInit != recurrentWeightInit || distribution != recurrentDistribution)
             if (enforceTrainingConfig)
                 throw new UnsupportedKerasConfigurationException(
                                 "Specifying different initialization for recurrent weights not supported.");
@@ -135,6 +144,8 @@ public KerasLstm(Map<String, Object> layerConfig, boolean enforceTrainingConfig)
                         .biasInit(0.0)
                         .l1(this.weightL1Regularization)
                         .l2(this.weightL2Regularization);
+        if (distribution != null)
+            builder.dist(distribution);
         if (biasConstraint != null)
             builder.constrainBias(biasConstraint);
         if (weightConstraint != null)

diff --git a/.../src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java b/.../src/main/java/org/deeplearning4j/nn/modelimport/keras/utils/KerasInitilizationUtils.java
@@ -87,7 +87,8 @@ public static Pair<WeightInit, Distribution> mapWeightInitialization(String kera
                     dist = new UniformDistribution(-scale, scale);
                 }
                 init = WeightInit.DISTRIBUTION;
-            } else if (kerasInit.equals(conf.getINIT_RANDOM_NORMAL()) ||
+            } else if (kerasInit.equals(conf.getINIT_NORMAL()) ||
+                    kerasInit.equals(conf.getINIT_RANDOM_NORMAL()) ||
                     kerasInit.equals(conf.getINIT_RANDOM_NORMAL_ALIAS())) {
                 if (kerasMajorVersion == 2) {
                     double mean = (double) initConfig.get(conf.getLAYER_FIELD_INIT_MEAN());
@@ -102,8 +103,8 @@ public static Pair<WeightInit, Distribution> mapWeightInitialization(String kera
                 init = WeightInit.DISTRIBUTION;
             } else if (kerasInit.equals(conf.getINIT_CONSTANT()) ||
                     kerasInit.equals(conf.getINIT_CONSTANT_ALIAS())) {
-                // FIXME: CONSTANT keras.initializers.Constant(value=0)
-                init = WeightInit.ZERO;
+                // TODO: CONSTANT keras.initializers.Constant(value=0)
+                init = WeightInit.ONES;
             } else if (kerasInit.equals(conf.getINIT_ORTHOGONAL()) ||
                     kerasInit.equals(conf.getINIT_ORTHOGONAL_ALIAS())) {
                 if (kerasMajorVersion == 2) {
@@ -189,7 +190,7 @@ public static Pair<WeightInit, Distribution> mapWeightInitialization(String kera
      * @throws InvalidKerasConfigurationException
      * @throws UnsupportedKerasConfigurationException
      */
-    public static WeightInit getWeightInitFromConfig(Map<String, Object> layerConfig, String initField,
+    public static Pair<WeightInit, Distribution> getWeightInitFromConfig(Map<String, Object> layerConfig, String initField,
                                                  boolean enforceTrainingConfig,
                                                  KerasLayerConfiguration conf,
                                                  int kerasMajorVersion)
@@ -222,7 +223,7 @@ public static WeightInit getWeightInitFromConfig(Map<String, Object> layerConfig
                 log.warn("Unknown weight initializer " + kerasInit + " (Using XAVIER instead).");
             }
         }
-        return init.getFirst();
+        return init;
     }
 
 }