Add gradient norm logging, fix tiny_cs for regression

Author: Lucas Zimmer

autoPyTorch/components/metrics/additional_logs.py

@@ -1,4 +1,3 @@
-
 class test_result():
     """Log the performance on the test set"""
     def __init__(self, autonet, X_test, Y_test):
@@ -10,4 +9,24 @@ def __call__(self, model, epochs):
         if self.Y_test is None or self.X_test is None:
             return float("nan")
 
-        return self.autonet.score(self.X_test, self.Y_test)
+        return self.autonet.score(self.X_test, self.Y_test)
+
+
+class gradient_norm():
+    """Log the norm of the gradients"""
+    def __init_(self):
+        pass
+
+    def __call__(self, network, epoch):
+        total_gradient = 0
+        n_params = 0
+
+        for p in list(filter(lambda p: p.grad is not None, network.parameters())):
+            total_gradient += p.grad.data.norm(2).item()
+            n_params += 1
+
+        # Prevent division through 0
+        if total_gradient==0:
+            n_params = 1
+
+        return total_gradient/n_params

autoPyTorch/core/presets/feature_regression/tiny_cs.txt

@@ -5,7 +5,7 @@ preprocessors=[truncated_svd]
 batch_loss_computation_techniques=[standard]
 imputation_strategies=[median]
 initialization_methods=[default]
-loss_modules=[cross_entropy_weighted]
+loss_modules=[l1_loss]
 normalization_strategies=[standardize]
 optimizer=[sgd]
-hyperparameter_search_space_updates=autoPyTorch/core/presets/tiny_cs_updates.txt
+hyperparameter_search_space_updates=autoPyTorch/core/presets/tiny_cs_updates.txt

autoPyTorch/pipeline/nodes/image/optimization_algorithm_no_timelimit.py

@@ -1,4 +1,3 @@
-
 import numpy as np
 import os
 import time
@@ -9,7 +8,6 @@
 import itertools
 import random
 
-
 import autoPyTorch.utils.thread_read_write as thread_read_write
 import datetime
 
@@ -75,7 +73,6 @@ def __init__(self, optimization_pipeline_nodes):
     def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, refit=None):
         res = None
 
-                
         config_space = self.pipeline.get_hyperparameter_search_space(**pipeline_config)
         config_space, constants = remove_constant_hyperparameter(config_space)
         config_space.seed(pipeline_config['random_seed'])
@@ -126,7 +123,6 @@ def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, refit=None):
             if task_id in [1, -1]:
                 self.run_optimization_algorithm(pipeline_config, config_space, constants, run_id, ns_host, ns_port, NS, task_id)
 
-
             res = self.parse_results(pipeline_config["result_logger_dir"])
 
         except Exception as e:

autoPyTorch/pipeline/nodes/optimization_algorithm.py

@@ -157,7 +157,7 @@ def predict(self, pipeline_config, X):
     def get_pipeline_config_options(self):
         options = [
             ConfigOption("run_id", default="0", type=str, info="Unique id for each run."),
-            ConfigOption("task_id", default=-1, type=int, info="ID for each worker, if you run AutoNet on a cluster. Set to -1, if you run it locally. "),
+            ConfigOption("task_id", default=-1, type=int, info="ID for each worker, if you run AutoNet on a cluster. Set to -1, if you run it locally."),
             ConfigOption("algorithm", default="bohb", type=str, choices=list(self.algorithms.keys()), info="Algorithm to use for config sampling."),
             ConfigOption("budget_type", default="time", type=str, choices=list(self.budget_types.keys())),
             ConfigOption("min_budget", default=lambda c: self.budget_types[c["budget_type"]].default_min_budget, type=float, depends=True, info="Min budget for fitting configurations."),