Update _Tests

Author: carefree0910

_Dist/NeuralNetworks/_Tests/AdvancedNN/Test.py

@@ -60,7 +60,7 @@ def block_test_pruner(generator, **kwargs):
 
 
 base_params = {
-    "model_param_settings": {"n_epoch": 40},
+    "model_param_settings": {"n_epoch": 40, "metric": "acc"},
     "model_structure_settings": {"hidden_units": [500, 500]}
 }
 basic = Basic(**base_params).fit(x, y, x_test, y_test, snapshot_ratio=0)
@@ -93,7 +93,7 @@ def block_test_pruner(generator, **kwargs):
 draw_acc(basic, wnd_dndf, wnd_dndf_pruned)
 
 advanced_params["data_info"]["numerical_idx"] = [True] * 500 + [False]
-advanced_params["model_structure_settings"]["pruner_params"] = {"prune_method": "surgery"}
+advanced_params["model_structure_settings"]["pruner_params"] = {"prune_method": "hard_prune"}
 basic, wnd_dndf, wnd_dndf_pruned = block_test(DataUtil.gen_noisy_linear, n_dim=500)
 draw_acc(basic, wnd_dndf, wnd_dndf_pruned, ylim=(0.8, 0.95), draw_train=False)
 

_Dist/NeuralNetworks/_Tests/DistNN/Test.py

@@ -14,8 +14,8 @@
     },
     "model_param_settings": {
         "metric": "acc",
-        "n_epoch": 5,
-        "max_epoch": 10
+        "n_epoch": 1,
+        "max_epoch": 2
     }
 }
 

_Dist/NeuralNetworks/_Tests/DistNN/TestOpenML.py

@@ -58,9 +58,7 @@ def download_data():
 def main():
     base_params = {
         "data_info": {},
-        "model_param_settings": {},
-        # "model_param_settings": {"n_epoch": 1, "max_epoch": 1},
-        # "model_structure_settings": {"use_wide_network": False, "use_pruner": False}
+        "model_param_settings": {}
     }
     config = tf.ConfigProto()
     config.gpu_options.allow_growth = True
@@ -74,8 +72,6 @@ def main():
         local_params["data_info"]["numerical_idx"] = numerical_idx
         DistAdvanced(**local_params).empirical_search(cv_rate=0.1, test_rate=0.1).k_random(
             K_RANDOM, cv_rate=0.1, test_rate=0.1)
-        # DistAdvanced(**local_params).k_random(K_RANDOM, cv_rate=0.1, test_rate=0.1)
-        # DistAdvanced(**local_params).fit()
 
 
 if __name__ == '__main__':

_Dist/NeuralNetworks/_Tests/Madelon/TestAdvancedNN.py

@@ -32,7 +32,7 @@ def block_evaluate():
 
 ylim = (0.5, 1.05)
 base_params = {
-    "model_param_settings": {"n_epoch": 200},
+    "model_param_settings": {"n_epoch": 200, "metric": "acc"},
     "model_structure_settings": {"hidden_units": [152, 153]}
 }
 basic = Basic(**base_params).fit(x, y, x_test, y_test, snapshot_ratio=0)

_Dist/NeuralNetworks/_Tests/Madelon/TestAutoNN.py

@@ -12,7 +12,7 @@
 
 base_params = {
     "name": "Madelon",
-    "model_param_settings": {"n_epoch": 200},
+    "model_param_settings": {"n_epoch": 200, "metric": "acc"},
     "model_structure_settings": {"hidden_units": [152, 153]}
 }
 basic = AutoBasic(**base_params).fit(x, y, x_test, y_test, snapshot_ratio=0)

_Dist/NeuralNetworks/_Tests/SVM/TestLinearSVM.py

@@ -8,10 +8,10 @@
 
 from Util.Util import DataUtil
 from _Dist.NeuralNetworks._Tests.TestUtil import draw_acc
-from _Dist.NeuralNetworks.f_AutoNN.NN import AutoAdvanced
+from _Dist.NeuralNetworks.f_AutoNN.DistNN import AutoAdvanced
 from _Dist.NeuralNetworks.b_TraditionalML.SVM import LinearSVM, AutoLinearSVM, DistLinearSVM
 
-base_params = {"model_param_settings": {"n_epoch": 30}}
+base_params = {"model_param_settings": {"n_epoch": 30, "metric": "acc"}}
 (x, y), (x_test, y_test) = DataUtil.gen_noisy_linear(n_dim=2, n_valid=2, test_ratio=0.01, one_hot=False)
 svm = LinearSVM(**deepcopy(base_params)).fit(
     x, y, x_test, y_test, snapshot_ratio=0).visualize2d(x_test, y_test)

_Dist/NeuralNetworks/_Tests/SVM/TestOpenML.py

@@ -0,0 +1,80 @@
+import os
+import sys
+root_path = os.path.abspath("./")
+if root_path not in sys.path:
+    sys.path.append(root_path)
+
+import copy
+import numpy as np
+import tensorflow as tf
+
+from openml import datasets
+
+from _Dist.NeuralNetworks.b_TraditionalML.SVM import DistLinearSVM
+
+GPU_ID = None
+K_RANDOM = 9
+IDS = [
+    38, 46, 179,
+    184, 389, 554,
+    772, 917, 1049,
+    1111, 1120, 1128,
+    293,
+]
+
+
+def swap(arr, i1, i2):
+    arr[..., i1], arr[..., i2] = arr[..., i2], arr[..., i1].copy()
+
+
+def download_data():
+    data_folder = "_Data"
+    idx_folder = os.path.join(data_folder, "idx")
+    if not os.path.isdir(data_folder):
+        os.makedirs(data_folder)
+    if not os.path.isdir(idx_folder):
+        os.makedirs(idx_folder)
+    for idx in IDS:
+        print("Downloading {}".format(idx))
+        data_file = os.path.join(data_folder, "{}.txt".format(idx))
+        idx_file = os.path.join(idx_folder, "{}.npy".format(idx))
+        if os.path.isfile(data_file) and os.path.isfile(idx_file):
+            continue
+        dataset = datasets.get_dataset(idx)
+        data, categorical_idx, names = dataset.get_data(
+            return_categorical_indicator=True,
+            return_attribute_names=True
+        )
+        data = data.toarray() if not isinstance(data, np.ndarray) else data
+        target_idx = names.index(dataset.default_target_attribute)
+        numerical_idx = ~np.array(categorical_idx)
+        swap(numerical_idx, target_idx, -1)
+        swap(data, target_idx, -1)
+        with open(data_file, "w") as file:
+            file.write("\n".join([" ".join(map(lambda n: str(n), line)) for line in data]))
+        np.save(idx_file, numerical_idx)
+
+
+def main():
+    base_params = {
+        "data_info": {},
+        "model_param_settings": {},
+        # "model_param_settings": {"n_epoch": 1, "max_epoch": 1},
+        # "model_structure_settings": {"use_wide_network": False, "use_pruner": False}
+    }
+    config = tf.ConfigProto()
+    config.gpu_options.allow_growth = True
+    if GPU_ID is not None:
+        os.environ["CUDA_VISIBLE_DEVICES"] = GPU_ID
+    base_params["model_param_settings"]["sess_config"] = config
+    for idx in IDS:
+        numerical_idx = np.load("_Data/idx/{}.npy".format(idx))
+        local_params = copy.deepcopy(base_params)
+        local_params["name"] = str(idx)
+        local_params["data_info"]["numerical_idx"] = numerical_idx
+        DistLinearSVM(**copy.deepcopy(local_params)).k_random(K_RANDOM, cv_rate=0.1, test_rate=0.1)
+
+
+if __name__ == '__main__':
+    download_data()
+    main()

_Dist/NeuralNetworks/_Tests/SVM/TestSVM.py

@@ -8,20 +8,13 @@
 
 from Util.Util import DataUtil
 from _Dist.NeuralNetworks._Tests.TestUtil import draw_acc
-from _Dist.NeuralNetworks.f_AutoNN.NN import AutoAdvanced
-from _Dist.NeuralNetworks.b_TraditionalML.SVM import SVM, AutoSVM, DistSVM
+from _Dist.NeuralNetworks.b_TraditionalML.SVM import SVM
+from _Dist.NeuralNetworks.f_AutoNN.DistNN import AutoAdvanced
 
-base_params = {"model_param_settings": {"n_epoch": 30}}
+base_params = {"model_param_settings": {"n_epoch": 30, "metric": "acc"}}
 (x, y), (x_test, y_test) = DataUtil.gen_noisy_linear(n_dim=2, n_valid=2, test_ratio=0.01, one_hot=False)
 svm = SVM(**deepcopy(base_params)).fit(
     x, y, x_test, y_test, snapshot_ratio=0).visualize2d(x_test, y_test)
 nn = AutoAdvanced("NoisyLinear", **deepcopy(base_params)).fit(
     x, y, x_test, y_test, snapshot_ratio=0).visualize2d(x_test, y_test)
 draw_acc(svm, nn)
-
-base_params["data_info"] = {"data_folder": "../_Data"}
-svm = AutoSVM("mushroom", **deepcopy(base_params)).fit(snapshot_ratio=0)
-nn = AutoAdvanced("mushroom", **deepcopy(base_params)).fit(snapshot_ratio=0)
-draw_acc(svm, nn)
-
-DistSVM("mushroom", **deepcopy(base_params)).k_random()