Update _Dist/NeuralNetworks

Author: carefree0910

_Dist/NeuralNetworks/Base.py

@@ -186,7 +186,6 @@ def __init__(self, name=None, model_param_settings=None, model_structure_setting
         self.log = {}
         self._name = name
         self._name_appendix = ""
-
         self._settings_initialized = False
 
         self._generator_base = Generator

_Dist/NeuralNetworks/NNUtil.py

@@ -678,7 +678,7 @@ def prune_w(self, w, w_abs, w_abs_mean, w_abs_std):
         self.cursor += 1
         with tf.name_scope("Prune"):
             if self.cond_placeholder is None:
-                log_w = tf.log(tf.maximum(self.eps, tf.abs(w) / (w_abs_mean * self.gamma)))
+                log_w = tf.log(tf.maximum(self.eps, w_abs / (w_abs_mean * self.gamma)))
                 if self.max_ratio > 0:
                     log_w = tf.minimum(self.max_ratio, self.beta * log_w)
                 self.masks.append(tf.maximum(self.alpha / self.beta * log_w, log_w))

_Dist/NeuralNetworks/d_Traditional2NN/Toolbox.py

@@ -172,3 +172,28 @@ def _transform(self):
         w2 *= max_route_length
         self._transform_ws = [w1, w2, w3]
         self._transform_bs = [b]
+
+
+if __name__ == '__main__':
+    train, test = np.load("train.npy"), np.load("test.npy")
+    x, y = train[..., :-1], train[..., -1]
+    x_test, y_test = test[..., :-1], test[..., -1]
+    # x_mean, x_std = x.mean(), x.std()
+    # x -= x_mean; x /= x_std
+    # x_test -= x_mean; x_test /= x_std
+    from _Dist.NeuralNetworks.e_AdvancedNN.NN import Advanced
+    Advanced(
+        name="madelon",
+        data_info={
+            "numerical_idx": [True] * 500 + [False],
+            "categorical_columns": []
+        },
+        model_param_settings={
+            "lr": 1e-3,
+            "activations": ["relu", "relu"]
+        }, model_structure_settings={
+            "use_pruner": False,
+            "use_wide_network": False,
+            "hidden_units": [152, 153]
+        }
+    ).fit(x, y, x_test, y_test, snapshot_ratio=1)

_Dist/NeuralNetworks/e_AdvancedNN/NN.py

@@ -61,10 +61,11 @@ def init_from_data(self, x, y, x_test, y_test, sample_weights, names):
 
     def init_model_param_settings(self):
         super(Advanced, self).init_model_param_settings()
-        self.dropout_keep_prob = self.model_param_settings.get("p_keep", 0.5)
-        self.use_batch_norm = self.model_param_settings.get("use_batch_norm", False)
+        self.dropout_keep_prob = self.model_param_settings.get("keep_prob", 0.5)
+        self.use_batch_norm = self.model_param_settings.get("use_batch_norm", True)
 
     def init_model_structure_settings(self):
+        self.hidden_units = self.model_structure_settings.get("hidden_units", None)
         self._deep_input = self.model_structure_settings.get("deep_input", "embedding_concat")
         self._wide_input = self.model_structure_settings.get("wide_input", "continuous")
         self.embedding_size = self.model_structure_settings.get("embedding_size", 8)
@@ -210,7 +211,8 @@ def _define_input_and_placeholder(self):
             self._deep_input = self._tfx
         else:
             self._deep_input = getattr(self, "_" + self._deep_input)
-        self._define_hidden_units()
+        if self.hidden_units is None:
+            self._define_hidden_units()
         self._settings = "{}_{}(dndf)_{}(prune)".format(
             self.hidden_units, self._dndf is not None, self._pruner is not None
         )