convlstm.py

import torch
import torch.nn as nn
import torch.nn.functional as F

import math

class ConvLSTMCell(nn.Module):
    def __init__(self, input_size, input_dim, hidden_dim, kernel_size, bias=True, activation=F.tanh, peephole=False, batchnorm=False):
        """
        Initialize ConvLSTM cell.
        Parameters
        ----------
        input_size: (int, int)
            Height and width of input tensor as (height, width).
        input_dim: int
            Number of channels of input tensor.
        hidden_dim: int
            Number of channels of hidden state.
        kernel_size: (int, int)
            Size of the convolutional kernel.
        bias: bool
            Whether or not to add the bias.
        """
        super(ConvLSTMCell, self).__init__()

        self.height, self.width = input_size
        self.input_dim          = input_dim
        self.hidden_dim         = hidden_dim

        self.kernel_size = kernel_size
        self.padding     = kernel_size[0] // 2, kernel_size[1] // 2
        self.bias        = bias
        self.activation  = activation
        self.peephole    = peephole
        self.batchnorm   = batchnorm

        if peephole:
            self.Wci = nn.Parameter(torch.FloatTensor(hidden_dim, self.height, self.width))
            self.Wcf = nn.Parameter(torch.FloatTensor(hidden_dim, self.height, self.width))
            self.Wco = nn.Parameter(torch.FloatTensor(hidden_dim, self.height, self.width))

        self.conv = nn.Conv2d(in_channels=self.input_dim + self.hidden_dim,
                              out_channels=4 * self.hidden_dim,
                              kernel_size=self.kernel_size,
                              padding=self.padding,
                              bias=self.bias)

        self.reset_parameters()

    def forward(self, input, prev_state):
        h_prev, c_prev = prev_state

        combined = torch.cat((input, h_prev), dim=1)  # concatenate along channel axis
        combined_conv = self.conv(combined)

        cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_dim, dim=1)

        if self.peephole:
            i = F.sigmoid(cc_i + self.Wci * c_prev)
            f = F.sigmoid(cc_f + self.Wcf * c_prev)
        else:
            i = F.sigmoid(cc_i)
            f = F.sigmoid(cc_f)

        g = self.activation(cc_g)
        c_cur = f * c_prev + i * g

        if self.peephole:
            o = F.sigmoid(cc_o + self.Wco * c_cur)
        else:
            o = F.sigmoid(cc_o)

        h_cur = o * self.activation(c_cur)

        return h_cur, c_cur

    def init_hidden(self, batch_size, cuda=True, device='cuda'):
        state = (torch.zeros(batch_size, self.hidden_dim, self.height, self.width),
                 torch.zeros(batch_size, self.hidden_dim, self.height, self.width))
        if cuda:
            state = (state[0].to(device), state[1].to(device))
        return state

    def reset_parameters(self):
        #self.conv.reset_parameters()
        nn.init.xavier_uniform_(self.conv.weight, gain=nn.init.calculate_gain('tanh'))
        self.conv.bias.data.zero_()

        if self.batchnorm:
            self.bn1.reset_parameters()
            self.bn2.reset_parameters()
        if self.peephole:
            std = 1. / math.sqrt(self.hidden_dim)
            self.Wci.data.uniform_(0,1)#(std=std)
            self.Wcf.data.uniform_(0,1)#(std=std)
            self.Wco.data.uniform_(0,1)#(std=std)


class ConvLSTM(nn.Module):
    def __init__(self, input_size, input_dim, hidden_dim, kernel_size, num_layers,
                 batch_first=False, bias=True, activation=F.tanh, peephole=False, batchnorm=False):
        super(ConvLSTM, self).__init__()

        self._check_kernel_size_consistency(kernel_size)

        # Make sure that both `kernel_size` and `hidden_dim` are lists having len == num_layers
        kernel_size = self._extend_for_multilayer(kernel_size, num_layers)
        hidden_dim  = self._extend_for_multilayer(hidden_dim, num_layers)
        activation  = self._extend_for_multilayer(activation, num_layers)

        if not len(kernel_size) == len(hidden_dim) == len(activation) == num_layers:
            raise ValueError('Inconsistent list length.')

        self.height, self.width = input_size

        self.input_dim  = input_dim
        self.hidden_dim = hidden_dim
        self.kernel_size = kernel_size
        self.num_layers = num_layers
        self.batch_first = batch_first
        self.bias = bias

        cell_list = []
        for i in range(0, self.num_layers):
            cur_input_dim = self.input_dim if i == 0 else self.hidden_dim[i-1]

            cell_list.append(ConvLSTMCell(input_size=(self.height, self.width),
                                          input_dim=cur_input_dim,
                                          hidden_dim=self.hidden_dim[i],
                                          kernel_size=self.kernel_size[i],
                                          bias=self.bias,
                                          activation=activation[i],
                                          peephole=peephole,
                                          batchnorm=batchnorm))

        self.cell_list = nn.ModuleList(cell_list)

        self.device = 'cpu'
        self.reset_parameters()

    def forward(self, input, hidden_state):
        """

        Parameters
        ----------
        input_tensor:
            5-D Tensor either of shape (t, b, c, h, w) or (b, t, c, h, w)
        hidden_state:
        Returns
        -------
        last_state_list, layer_output
        """
        cur_layer_input = torch.unbind(input, dim=int(self.batch_first))

        if not hidden_state:
            hidden_state = self.get_init_states(cur_layer_input[0].size(int(not self.batch_first)))

        seq_len = len(cur_layer_input)

        layer_output_list = []
        last_state_list   = []

        for layer_idx in range(self.num_layers):
            h, c = hidden_state[layer_idx]
            output_inner = []
            for t in range(seq_len):
                h, c = self.cell_list[layer_idx](input=cur_layer_input[t],
                                                 prev_state=[h, c])
                output_inner.append(h)

            cur_layer_input = output_inner
            last_state_list.append((h, c))

        layer_output = torch.stack(output_inner, dim=int(self.batch_first))

        return layer_output, last_state_list

    def reset_parameters(self):
        for c in self.cell_list:
            c.reset_parameters()

    def get_init_states(self, batch_size, cuda=True, device='cuda'):
        init_states = []
        for i in range(self.num_layers):
            init_states.append(self.cell_list[i].init_hidden(batch_size, cuda, device))
        return init_states

    @staticmethod
    def _check_kernel_size_consistency(kernel_size):
        if not (isinstance(kernel_size, tuple) or (isinstance(kernel_size, list)
            and all([isinstance(elem, tuple) for elem in kernel_size]))):
            raise ValueError('`Kernel_size` must be tuple or list of tuples')

    @staticmethod
    def _extend_for_multilayer(param, num_layers):
        if not isinstance(param, list):
            param = [param] * num_layers
        return param