Cocktail hotfixes (#245)

* Fixes for the development branch and regularization cocktails

* Update implementation

* Fix unit tests temporarily

* Implementation update and bug fixes

* Removing unecessary code

* Addressing Ravin's comments

Author: ArlindKadra

autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py

@@ -64,7 +64,8 @@ def _add_group(self, in_features: int, out_features: int,
             out_features (int): output dimensionality for the current block
             blocks_per_group (int): Number of ResNet per group
             last_block_index (int): block index for shake regularization
-            dropout (bool): whether or not use dropout
+            dropout (None, float): dropout value for the group. If none,
+                no dropout is applied.
         """
         blocks = list()
         for i in range(blocks_per_group):
@@ -290,13 +291,22 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module:
             if self.config['use_batch_norm']:
                 layers.append(nn.BatchNorm1d(in_features))
             layers.append(self.activation())
+        else:
+            # if start norm is not None and skip connection is None
+            # we will never apply the start_norm for the first layer in the block,
+            # which is why we should account for this case.
+            if not self.config['use_skip_connection']:
+                if self.config['use_batch_norm']:
+                    layers.append(nn.BatchNorm1d(in_features))
+                layers.append(self.activation())
+
         layers.append(nn.Linear(in_features, out_features))
 
         if self.config['use_batch_norm']:
             layers.append(nn.BatchNorm1d(out_features))
         layers.append(self.activation())
 
-        if self.config["use_dropout"]:
+        if self.dropout is not None:
             layers.append(nn.Dropout(self.dropout))
         layers.append(nn.Linear(out_features, out_features))
 
@@ -321,6 +331,7 @@ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
             if self.config["use_skip_connection"]:
                 residual = self.shortcut(x)
 
+        # TODO make the below code better
         if self.config["use_skip_connection"]:
             if self.config["multi_branch_choice"] == 'shake-shake':
                 x1 = self.layers(x)

autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py

@@ -31,11 +31,13 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential:
         out_features = self.config["output_dim"]
 
         # use the get_shaped_neuron_counts to update the number of units
-        neuron_counts = get_shaped_neuron_counts(self.config['resnet_shape'],
-                                                 in_features,
-                                                 out_features,
-                                                 self.config['max_units'],
-                                                 self.config['num_groups'] + 2)[:-1]
+        neuron_counts = get_shaped_neuron_counts(
+            self.config['resnet_shape'],
+            in_features,
+            out_features,
+            self.config['max_units'],
+            self.config['num_groups'] + 2,
+        )[:-1]
         self.config.update(
             {"num_units_%d" % (i): num for i, num in enumerate(neuron_counts)}
         )
@@ -45,12 +47,12 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential:
             # n_units for the architecture, since, it is mostly implemented for the
             # output layer, which is part of the head and not of the backbone.
             dropout_shape = get_shaped_neuron_counts(
-                self.config['dropout_shape'], 0, 0, 1000, self.config['num_groups']
-            )
-
-            dropout_shape = [
-                dropout / 1000 * self.config["max_dropout"] for dropout in dropout_shape
-            ]
+                self.config['dropout_shape'],
+                0,
+                0,
+                self.config["max_dropout"],
+                self.config['num_groups'] + 1,
+            )[:-1]
 
             self.config.update(
                 {"dropout_%d" % (i + 1): dropout for i, dropout in enumerate(dropout_shape)}

autoPyTorch/pipeline/components/setup/network_head/no_head.py

@@ -20,7 +20,7 @@ class NoHead(NetworkHeadComponent):
     """
 
     def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...]) -> nn.Module:
-        layers = [nn.Flatten()]
+        layers = []
         in_features = np.prod(input_shape).item()
         out_features = np.prod(output_shape).item()
         layers.append(_activations[self.config["activation"]]())
@@ -34,8 +34,8 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[
             'shortname': 'NoHead',
             'name': 'NoHead',
             'handles_tabular': True,
-            'handles_image': True,
-            'handles_time_series': True,
+            'handles_image': False,
+            'handles_time_series': False,
         }
 
     @staticmethod

test/test_pipeline/components/setup/test_setup.py

@@ -501,8 +501,7 @@ def test_dropout(self, resnet_shape):
 class TestNetworkHead:
     def test_all_heads_available(self):
         network_head_choice = NetworkHeadChoice(dataset_properties={})
-
-        assert len(network_head_choice.get_components().keys()) == 2
+        assert len(network_head_choice.get_components().keys()) == 3
 
     @pytest.mark.parametrize('task_type_input_output_shape', [(constants.IMAGE_CLASSIFICATION, (3, 64, 64), (5,)),
                                                               (constants.IMAGE_REGRESSION, (3, 64, 64), (1,)),
@@ -520,7 +519,9 @@ def test_dummy_forward_backward_pass(self, task_type_input_output_shape):
         if task_type in constants.CLASSIFICATION_TASKS:
             dataset_properties["num_classes"] = output_shape[0]
 
-        cs = network_head_choice.get_hyperparameter_search_space(dataset_properties=dataset_properties)
+        cs = network_head_choice.get_hyperparameter_search_space(
+            dataset_properties=dataset_properties,
+        )
         # test 10 random configurations
         for _ in range(10):
             config = cs.sample_configuration()