Skip to content

Network Cleanup #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Feb 2, 2021
Merged

Network Cleanup #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
9d50cb6
removed old supported_tasks dictionary from heads, added some docstri…
bastiscode Feb 1, 2021
b7c8773
removed old supported_tasks attribute and updated doc strings in base…
bastiscode Feb 1, 2021
725faf2
removed old supported_tasks attribute from network backbones
bastiscode Feb 1, 2021
740a604
put time series backbones in separate files, add doc strings and refa…
bastiscode Feb 1, 2021
b727016
split image networks into separate files, add doc strings and refacto…
bastiscode Feb 1, 2021
bc77ca3
fix typo
bastiscode Feb 1, 2021
f8de549
add an intial simple backbone test similar to the network head test
bastiscode Feb 1, 2021
480b8ea
fix flake8
bastiscode Feb 1, 2021
f461c7e
fixed imports in backbones and heads
bastiscode Feb 2, 2021
cab8f83
added new network backbone and head tests
bastiscode Feb 2, 2021
ab2f5e9
enabled tests for adding custom backbones and heads, added required p…
bastiscode Feb 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 122 additions & 0 deletions autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
from typing import Any, Dict, List, Optional, Tuple

import ConfigSpace as CS
from ConfigSpace.configuration_space import ConfigurationSpace
from ConfigSpace.hyperparameters import (
CategoricalHyperparameter,
UniformIntegerHyperparameter
)

from torch import nn

from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent
from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations


class ConvNetImageBackbone(NetworkBackboneComponent):
"""
Standard Convolutional Neural Network backbone for images
"""

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
self.bn_args = {"eps": 1e-5, "momentum": 0.1}

def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
return cw, ch

def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
layers.append(nn.Conv2d(in_filters, out_filters,
kernel_size=self.config["conv_kernel_size"],
stride=self.config["conv_kernel_stride"],
padding=self.config["conv_kernel_padding"]))
layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
layers.append(_activations[self.config["activation"]]())
layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))

def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
channels, iw, ih = input_shape
layers: List[nn.Module] = []
init_filter = self.config["conv_init_filters"]
self._add_layer(layers, channels, init_filter)

cw, ch = self._get_layer_size(iw, ih)
for i in range(2, self.config["num_layers"] + 1):
cw, ch = self._get_layer_size(cw, ch)
if cw == 0 or ch == 0:
break
self._add_layer(layers, init_filter, init_filter * 2)
init_filter *= 2
backbone = nn.Sequential(*layers)
self.backbone = backbone
return backbone

@staticmethod
def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
return {
'shortname': 'ConvNetImageBackbone',
'name': 'ConvNetImageBackbone',
'handles_tabular': False,
'handles_image': True,
'handles_time_series': False,
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
num_layers: Tuple[Tuple, int] = ((2, 8), 4),
num_init_filters: Tuple[Tuple, int] = ((16, 64), 32),
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
kernel_size: Tuple[Tuple, int] = ((3, 5), 3),
stride: Tuple[Tuple, int] = ((1, 3), 1),
padding: Tuple[Tuple, int] = ((2, 3), 2),
pool_size: Tuple[Tuple, int] = ((2, 3), 2)
) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

min_num_layers, max_num_layers = num_layers[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
lower=min_num_layers,
upper=max_num_layers,
default_value=num_layers[1]))

cs.add_hyperparameter(CategoricalHyperparameter('activation',
choices=activation[0],
default_value=activation[1]))

min_init_filters, max_init_filters = num_init_filters[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
lower=min_init_filters,
upper=max_init_filters,
default_value=num_init_filters[1]))

min_kernel_size, max_kernel_size = kernel_size[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
lower=min_kernel_size,
upper=max_kernel_size,
default_value=kernel_size[1]))

min_stride, max_stride = stride[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
lower=min_stride,
upper=max_stride,
default_value=stride[1]))

min_padding, max_padding = padding[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
lower=min_padding,
upper=max_padding,
default_value=padding[1]))

min_pool_size, max_pool_size = pool_size[0]
cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
lower=min_pool_size,
upper=max_pool_size,
default_value=pool_size[1]))

return cs
149 changes: 35 additions & 114 deletions ...omponents/setup/network_backbone/image.py → .../network_backbone/DenseNetImageBackone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import logging
import math
from collections import OrderedDict
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, Optional, Tuple

import ConfigSpace as CS
from ConfigSpace.configuration_space import ConfigurationSpace
Expand All @@ -16,103 +15,7 @@
from torch.nn import functional as F

from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent

_activations: Dict[str, nn.Module] = {
"relu": nn.ReLU,
"tanh": nn.Tanh,
"sigmoid": nn.Sigmoid
}


class ConvNetImageBackbone(NetworkBackboneComponent):
supported_tasks = {"image_classification", "image_regression"}

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
self.bn_args = {"eps": 1e-5, "momentum": 0.1}

def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
// self.config["conv_kernel_stride"]) + 1
cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
return cw, ch

def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
layers.append(nn.Conv2d(in_filters, out_filters,
kernel_size=self.config["conv_kernel_size"],
stride=self.config["conv_kernel_stride"],
padding=self.config["conv_kernel_padding"]))
layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
layers.append(_activations[self.config["activation"]]())
layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))

def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
channels, iw, ih = input_shape
layers: List[nn.Module] = []
init_filter = self.config["conv_init_filters"]
self._add_layer(layers, channels, init_filter)

cw, ch = self._get_layer_size(iw, ih)
for i in range(2, self.config["num_layers"] + 1):
cw, ch = self._get_layer_size(cw, ch)
if cw == 0 or ch == 0:
logging.info("> reduce network size due to too small layers.")
break
self._add_layer(layers, init_filter, init_filter * 2)
init_filter *= 2
backbone = nn.Sequential(*layers)
self.backbone = backbone
return backbone

@staticmethod
def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
return {
'shortname': 'ConvNetImageBackbone',
'name': 'ConvNetImageBackbone',
'handles_tabular': False,
'handles_image': True,
'handles_time_series': False,
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
min_num_layers: int = 2,
max_num_layers: int = 5,
min_init_filters: int = 16,
max_init_filters: int = 64,
min_kernel_size: int = 2,
max_kernel_size: int = 5,
min_stride: int = 1,
max_stride: int = 3,
min_padding: int = 2,
max_padding: int = 3,
min_pool_size: int = 2,
max_pool_size: int = 3) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
lower=min_num_layers,
upper=max_num_layers))
cs.add_hyperparameter(CategoricalHyperparameter('activation',
choices=list(_activations.keys())))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
lower=min_init_filters,
upper=max_init_filters))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
lower=min_kernel_size,
upper=max_kernel_size))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
lower=min_stride,
upper=max_stride))
cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
lower=min_padding,
upper=max_padding))
cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
lower=min_pool_size,
upper=max_pool_size))
return cs
from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations


class _DenseLayer(nn.Sequential):
Expand Down Expand Up @@ -177,7 +80,9 @@ def __init__(self,


class DenseNetBackbone(NetworkBackboneComponent):
supported_tasks = {"image_classification", "image_regression"}
"""
Dense Net Backbone for images (see https://arxiv.org/pdf/1608.06993.pdf)
"""

def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
Expand Down Expand Up @@ -247,39 +152,55 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[
}

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
min_growth_rate: int = 12,
max_growth_rate: int = 40,
min_num_blocks: int = 3,
max_num_blocks: int = 4,
min_num_layers: int = 4,
max_num_layers: int = 64) -> ConfigurationSpace:
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
num_blocks: Tuple[Tuple, int] = ((3, 4), 3),
num_layers: Tuple[Tuple, int] = ((4, 64), 16),
growth_rate: Tuple[Tuple, int] = ((12, 40), 20),
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
dropout: Tuple[Tuple, float] = ((0, 0.5), 0.2)
) -> ConfigurationSpace:
cs = CS.ConfigurationSpace()

min_growth_rate, max_growth_rate = growth_rate[0]
growth_rate_hp = UniformIntegerHyperparameter('growth_rate',
lower=min_growth_rate,
upper=max_growth_rate)
upper=max_growth_rate,
default_value=growth_rate[1])
cs.add_hyperparameter(growth_rate_hp)

min_num_blocks, max_num_blocks = num_blocks[0]
blocks_hp = UniformIntegerHyperparameter('blocks',
lower=min_num_blocks,
upper=max_num_blocks)
upper=max_num_blocks,
default_value=num_blocks[1])
cs.add_hyperparameter(blocks_hp)

activation_hp = CategoricalHyperparameter('activation',
choices=list(_activations.keys()))
choices=activation[0],
default_value=activation[1])
cs.add_hyperparameter(activation_hp)

use_dropout = CategoricalHyperparameter('use_dropout', choices=[True, False])
use_dropout = CategoricalHyperparameter('use_dropout',
choices=use_dropout[0],
default_value=use_dropout[1])

min_dropout, max_dropout = dropout[0]
dropout = UniformFloatHyperparameter('dropout',
lower=0.0,
upper=1.0)
lower=min_dropout,
upper=max_dropout,
default_value=dropout[1])

cs.add_hyperparameters([use_dropout, dropout])
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))

for i in range(1, max_num_blocks + 1):
min_num_layers, max_num_layers = num_layers[0]
layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i,
lower=min_num_layers,
upper=max_num_layers)
upper=max_num_layers,
default_value=num_layers[1])
cs.add_hyperparameter(layer_hp)

if i > min_num_blocks:
Expand Down
Loading