Skip to content

Commit

Permalink
added layer_noise and dataset_noise
Browse files Browse the repository at this point in the history
  • Loading branch information
Hycis authored and Hycis committed Nov 6, 2014
1 parent f94d311 commit f5dbf92
Show file tree
Hide file tree
Showing 17 changed files with 1,224 additions and 384 deletions.
113 changes: 81 additions & 32 deletions hps/models.py → hps/AE.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
from pynet.train_object import TrainObject
from pynet.cost import Cost
import pynet.datasets.preprocessor as preproc

import pynet.datasets.dataset_noise as noisy
import pynet.layer_noise as layer_noise
import cPickle
import os

Expand Down Expand Up @@ -54,12 +55,25 @@ def build_log(self, save_to_database=None, id=None):
def build_dataset(self):
dataset = None

preprocessor = None if self.state.dataset.preprocessor is None else \
getattr(preproc, self.state.dataset.preprocessor)()
preprocessor = None if self.state.dataset.preprocessor.type is None else \
getattr(preproc, self.state.dataset.preprocessor.type)()

# if self.state.dataset.noise.type == 'BlackOut' or self.state.dataset.noise.type == 'MaskOut':
# noise = None if self.state.dataset.noise.type is None else \
# getattr(noisy, self.state.dataset.noise.type)(ratio=self.state.dataset.noise.ratio)
# else:
# noise = getattr(noisy, self.state.dataset.noise.type)()
noise = None if self.state.dataset.dataset_noise.type is None else \
getattr(noisy, self.state.dataset.dataset_noise.type)()

if self.state.dataset.preprocessor.type == 'Scale':
preprocessor.max = self.state.dataset.preprocessor.global_max
preprocessor.min = self.state.dataset.preprocessor.global_min

if self.state.dataset.type == 'Mnist':
dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
noise = noise,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
Expand All @@ -77,6 +91,7 @@ def build_dataset(self):
target_size = self.state.dataset.feature_size,
train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
noise = noise,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
Expand All @@ -86,6 +101,7 @@ def build_dataset(self):
dataset = getattr(spec, self.state.dataset.type)(
train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
noise = noise,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
Expand All @@ -103,6 +119,7 @@ def build_dataset(self):
target_size = self.state.dataset.feature_size,
train_valid_test_ratio = self.state.dataset.train_valid_test_ratio,
preprocessor = preprocessor,
noise = noise,
batch_size = self.state.dataset.batch_size,
num_batches = self.state.dataset.num_batches,
iter_class = self.state.dataset.iter_class,
Expand All @@ -127,17 +144,20 @@ def build_learning_rule(self):


def build_one_hid_model(self, input_dim):
model = AutoEncoder(input_dim = input_dim, rand_seed=self.state.model.rand_seed)
model = AutoEncoder(input_dim=input_dim, rand_seed=self.state.model.rand_seed)

hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim,
name=self.state.hidden1.name,
dropout_below=self.state.hidden1.dropout_below,
blackout_below=self.state.hidden1.blackout_below)
noise=None if self.state.hidden1.layer_noise is None else \
getattr(layer_noise, self.state.hidden1.layer_noise)())
# blackout_below=self.state.hidden1.blackout_below)
model.add_encode_layer(hidden1)
h1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim,
name=self.state.h1_mirror.name,
W=hidden1.W.T,
dropout_below=self.state.h1_mirror.dropout_below,
blackout_below=self.state.h1_mirror.blackout_below)
dropout_below=self.state.h1_mirror.dropout_below)
# blackout_below=self.state.h1_mirror.blackout_below)
model.add_decode_layer(h1_mirror)
return model

Expand All @@ -146,12 +166,14 @@ def build_one_hid_model_no_transpose(self, input_dim):
hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim,
name=self.state.hidden1.name,
dropout_below=self.state.hidden1.dropout_below,
blackout_below=self.state.hidden1.blackout_below)
noise=None if self.state.hidden1.layer_noise is None else \
getattr(layer_noise, self.state.hidden1.layer_noise)())
# blackout_below=self.state.hidden1.blackout_below)
model.add_encode_layer(hidden1)
h1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim,
name=self.state.h1_mirror.name,
dropout_below=self.state.h1_mirror.dropout_below,
blackout_below=self.state.h1_mirror.blackout_below)
dropout_below=self.state.h1_mirror.dropout_below)
# blackout_below=self.state.h1_mirror.blackout_below)
model.add_decode_layer(h1_mirror)
return model

Expand All @@ -160,26 +182,30 @@ def build_two_hid_model(self, input_dim):
hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim,
name=self.state.hidden1.name,
dropout_below=self.state.hidden1.dropout_below,
blackout_below=self.state.hidden1.blackout_below)
noise=None if self.state.hidden1.layer_noise is None else \
getattr(layer_noise, self.state.hidden1.layer_noise)())
# blackout_below=self.state.hidden1.blackout_below)
model.add_encode_layer(hidden1)

hidden2 = getattr(layer, self.state.hidden2.type)(dim=self.state.hidden2.dim,
name=self.state.hidden2.name,
dropout_below=self.state.hidden2.dropout_below,
blackout_below=self.state.hidden2.blackout_below)
noise=None if self.state.hidden2.layer_noise is None else \
getattr(layer_noise, self.state.hidden2.layer_noise)())
# blackout_below=self.state.hidden2.blackout_below)
model.add_encode_layer(hidden2)

hidden2_mirror = getattr(layer, self.state.h2_mirror.type)(dim=self.state.hidden1.dim,
name=self.state.h2_mirror.name,
dropout_below=self.state.h2_mirror.dropout_below,
blackout_below=self.state.h2_mirror.blackout_below,
# blackout_below=self.state.h2_mirror.blackout_below,
W = hidden2.W.T)
model.add_decode_layer(hidden2_mirror)

hidden1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim,
name=self.state.h1_mirror.name,
dropout_below=self.state.h1_mirror.dropout_below,
blackout_below=self.state.h1_mirror.blackout_below,
# blackout_below=self.state.h1_mirror.blackout_below,
W = hidden1.W.T)
model.add_decode_layer(hidden1_mirror)
return model
Expand All @@ -191,8 +217,10 @@ def build_database(self, dataset, learning_rule, model):
'max_col_norm' : learning_rule.max_col_norm,
'Weight_Init_Seed' : model.rand_seed,
'Dropout_Below' : str([layer.dropout_below for layer in model.layers]),
'Blackout_Below' : str([layer.blackout_below for layer in model.layers]),
# 'Blackout_Below' : str([layer.blackout_below for layer in model.layers]),
'Batch_Size' : dataset.batch_size,
'Dataset_Noise' : dataset.noise.__class__.__name__,
'Layer_Noise' : str([layer.noise.__class__.__name__ for layer in model.layers]),
'nblocks' : dataset.nblocks(),
'Layer_Types' : str([layer.__class__.__name__ for layer in model.layers]),
'Layer_Dim' : str([layer.dim for layer in model.layers]),
Expand Down Expand Up @@ -287,6 +315,9 @@ def run(self):
dataset = self.build_dataset()
learning_rule = self.build_learning_rule()

# import pdb
# pdb.set_trace()

if self.state.num_layers == 1:
model = self.build_one_hid_model(dataset.feature_size())
elif self.state.num_layers == 2:
Expand All @@ -297,6 +328,7 @@ def run(self):
database = self.build_database(dataset, learning_rule, model)
log = self.build_log(database)

dataset.log = log

train_obj = TrainObject(log = log,
dataset = dataset,
Expand All @@ -305,9 +337,9 @@ def run(self):

train_obj.run()

# fine tuning
log.info("fine tuning")
log.info("Fine Tuning")
train_obj.model.layers[0].dropout_below = None
# train_obj.model.layers[0].blackout_below = None
train_obj.setup()
train_obj.run()

Expand All @@ -317,7 +349,7 @@ def __init__(self, state):
self.state = state


def build_model(self, input_dim):
def build_model(self):
with open(os.environ['PYNET_SAVE_PATH'] + '/log/'
+ self.state.hidden1.model + '/model.pkl') as f1:
model = cPickle.load(f1)
Expand All @@ -328,8 +360,12 @@ def run(self):
dataset = self.build_dataset()
learning_rule = self.build_learning_rule()

model = self.build_model(dataset.feature_size())
model.layers[0].dropout_below = self.state.hidden1.dropout_below
model = self.build_model()

if self.state.fine_tuning_only:
model.layers[0].dropout_below = None
# model.layers[0].blackout_below = None
print "Fine Tuning Only"

if self.state.log.save_to_database_name:
database = self.build_database(dataset, learning_rule, model)
Expand All @@ -342,10 +378,13 @@ def run(self):
model = model)

train_obj.run()
log.info("fine tuning")
train_obj.model.layers[0].dropout_below = None
train_obj.setup()
train_obj.run()

if not self.state.fine_tuning_only:
log.info("..Fine Tuning after Noisy Training")
train_obj.model.layers[0].dropout_below = None
# train_obj.model.layers[0].blackout_below = None
train_obj.setup()
train_obj.run()



Expand Down Expand Up @@ -390,7 +429,7 @@ def run(self):
learning_rule = learning_rule,
model = model)

log.info("fine tuning")
log.info("Fine Tuning")
train_obj.model.layers[0].dropout_below = None
train_obj.model.layers[1].dropout_below = None
train_obj.run()
Expand Down Expand Up @@ -430,7 +469,16 @@ def run(self):
learning_rule = self.build_learning_rule()

model = self.build_model(dataset.feature_size())
model.layers[0].dropout_below = self.state.hidden1.dropout_below

if not self.state.fine_tuning_only:
model.layers[0].dropout_below = self.state.hidden1.dropout_below
# model.layers[0].blackout_below = self.state.hidden1.blackout_below

model.layers[1].dropout_below = self.state.hidden2.dropout_below
# model.layers[1].blackout_below = self.state.hidden2.blackout_below

model.layers[2].dropout_below = self.state.hidden3.dropout_below
# model.layers[2].blackout_below = self.state.hidden3.blackout_below

if self.state.log.save_to_database_name:
database = self.build_database(dataset, learning_rule, model)
Expand All @@ -444,14 +492,15 @@ def run(self):
learning_rule = learning_rule,
model = model)

# train_obj.run()

# fine tuning
log.info("fine tuning")
# train_obj.model.layers[0].dropout_below = None
# train_obj.setup()
train_obj.run()

if not self.state.fine_tuning_only:
log.info("..Fine Tuning after Noisy Training")
train_obj.model.layers[0].dropout_below = None
# train_obj.model.layers[0].blackout_below = None
train_obj.setup()
train_obj.run()


class Laura_Two_Layers_No_Transpose(AE):

Expand Down
Loading

0 comments on commit f5dbf92

Please sign in to comment.