diff --git a/hps/models.py b/hps/AE.py similarity index 80% rename from hps/models.py rename to hps/AE.py index ad1f480..b3afc8a 100644 --- a/hps/models.py +++ b/hps/AE.py @@ -13,7 +13,8 @@ from pynet.train_object import TrainObject from pynet.cost import Cost import pynet.datasets.preprocessor as preproc - +import pynet.datasets.dataset_noise as noisy +import pynet.layer_noise as layer_noise import cPickle import os @@ -54,12 +55,25 @@ def build_log(self, save_to_database=None, id=None): def build_dataset(self): dataset = None - preprocessor = None if self.state.dataset.preprocessor is None else \ - getattr(preproc, self.state.dataset.preprocessor)() + preprocessor = None if self.state.dataset.preprocessor.type is None else \ + getattr(preproc, self.state.dataset.preprocessor.type)() + + # if self.state.dataset.noise.type == 'BlackOut' or self.state.dataset.noise.type == 'MaskOut': + # noise = None if self.state.dataset.noise.type is None else \ + # getattr(noisy, self.state.dataset.noise.type)(ratio=self.state.dataset.noise.ratio) + # else: + # noise = getattr(noisy, self.state.dataset.noise.type)() + noise = None if self.state.dataset.dataset_noise.type is None else \ + getattr(noisy, self.state.dataset.dataset_noise.type)() + + if self.state.dataset.preprocessor.type == 'Scale': + preprocessor.max = self.state.dataset.preprocessor.global_max + preprocessor.min = self.state.dataset.preprocessor.global_min if self.state.dataset.type == 'Mnist': dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, + noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, @@ -77,6 +91,7 @@ def build_dataset(self): target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, + noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, @@ -86,6 +101,7 @@ def build_dataset(self): dataset = getattr(spec, self.state.dataset.type)( train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, + noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, @@ -103,6 +119,7 @@ def build_dataset(self): target_size = self.state.dataset.feature_size, train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, preprocessor = preprocessor, + noise = noise, batch_size = self.state.dataset.batch_size, num_batches = self.state.dataset.num_batches, iter_class = self.state.dataset.iter_class, @@ -127,17 +144,20 @@ def build_learning_rule(self): def build_one_hid_model(self, input_dim): - model = AutoEncoder(input_dim = input_dim, rand_seed=self.state.model.rand_seed) + model = AutoEncoder(input_dim=input_dim, rand_seed=self.state.model.rand_seed) + hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim, name=self.state.hidden1.name, dropout_below=self.state.hidden1.dropout_below, - blackout_below=self.state.hidden1.blackout_below) + noise=None if self.state.hidden1.layer_noise is None else \ + getattr(layer_noise, self.state.hidden1.layer_noise)()) + # blackout_below=self.state.hidden1.blackout_below) model.add_encode_layer(hidden1) h1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim, name=self.state.h1_mirror.name, W=hidden1.W.T, - dropout_below=self.state.h1_mirror.dropout_below, - blackout_below=self.state.h1_mirror.blackout_below) + dropout_below=self.state.h1_mirror.dropout_below) + # blackout_below=self.state.h1_mirror.blackout_below) model.add_decode_layer(h1_mirror) return model @@ -146,12 +166,14 @@ def build_one_hid_model_no_transpose(self, input_dim): hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim, name=self.state.hidden1.name, dropout_below=self.state.hidden1.dropout_below, - blackout_below=self.state.hidden1.blackout_below) + noise=None if self.state.hidden1.layer_noise is None else \ + getattr(layer_noise, self.state.hidden1.layer_noise)()) + # blackout_below=self.state.hidden1.blackout_below) model.add_encode_layer(hidden1) h1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim, name=self.state.h1_mirror.name, - dropout_below=self.state.h1_mirror.dropout_below, - blackout_below=self.state.h1_mirror.blackout_below) + dropout_below=self.state.h1_mirror.dropout_below) + # blackout_below=self.state.h1_mirror.blackout_below) model.add_decode_layer(h1_mirror) return model @@ -160,26 +182,30 @@ def build_two_hid_model(self, input_dim): hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim, name=self.state.hidden1.name, dropout_below=self.state.hidden1.dropout_below, - blackout_below=self.state.hidden1.blackout_below) + noise=None if self.state.hidden1.layer_noise is None else \ + getattr(layer_noise, self.state.hidden1.layer_noise)()) + # blackout_below=self.state.hidden1.blackout_below) model.add_encode_layer(hidden1) hidden2 = getattr(layer, self.state.hidden2.type)(dim=self.state.hidden2.dim, name=self.state.hidden2.name, dropout_below=self.state.hidden2.dropout_below, - blackout_below=self.state.hidden2.blackout_below) + noise=None if self.state.hidden2.layer_noise is None else \ + getattr(layer_noise, self.state.hidden2.layer_noise)()) + # blackout_below=self.state.hidden2.blackout_below) model.add_encode_layer(hidden2) hidden2_mirror = getattr(layer, self.state.h2_mirror.type)(dim=self.state.hidden1.dim, name=self.state.h2_mirror.name, dropout_below=self.state.h2_mirror.dropout_below, - blackout_below=self.state.h2_mirror.blackout_below, + # blackout_below=self.state.h2_mirror.blackout_below, W = hidden2.W.T) model.add_decode_layer(hidden2_mirror) hidden1_mirror = getattr(layer, self.state.h1_mirror.type)(dim=input_dim, name=self.state.h1_mirror.name, dropout_below=self.state.h1_mirror.dropout_below, - blackout_below=self.state.h1_mirror.blackout_below, + # blackout_below=self.state.h1_mirror.blackout_below, W = hidden1.W.T) model.add_decode_layer(hidden1_mirror) return model @@ -191,8 +217,10 @@ def build_database(self, dataset, learning_rule, model): 'max_col_norm' : learning_rule.max_col_norm, 'Weight_Init_Seed' : model.rand_seed, 'Dropout_Below' : str([layer.dropout_below for layer in model.layers]), - 'Blackout_Below' : str([layer.blackout_below for layer in model.layers]), + # 'Blackout_Below' : str([layer.blackout_below for layer in model.layers]), 'Batch_Size' : dataset.batch_size, + 'Dataset_Noise' : dataset.noise.__class__.__name__, + 'Layer_Noise' : str([layer.noise.__class__.__name__ for layer in model.layers]), 'nblocks' : dataset.nblocks(), 'Layer_Types' : str([layer.__class__.__name__ for layer in model.layers]), 'Layer_Dim' : str([layer.dim for layer in model.layers]), @@ -287,6 +315,9 @@ def run(self): dataset = self.build_dataset() learning_rule = self.build_learning_rule() + # import pdb + # pdb.set_trace() + if self.state.num_layers == 1: model = self.build_one_hid_model(dataset.feature_size()) elif self.state.num_layers == 2: @@ -297,6 +328,7 @@ def run(self): database = self.build_database(dataset, learning_rule, model) log = self.build_log(database) + dataset.log = log train_obj = TrainObject(log = log, dataset = dataset, @@ -305,9 +337,9 @@ def run(self): train_obj.run() - # fine tuning - log.info("fine tuning") + log.info("Fine Tuning") train_obj.model.layers[0].dropout_below = None + # train_obj.model.layers[0].blackout_below = None train_obj.setup() train_obj.run() @@ -317,7 +349,7 @@ def __init__(self, state): self.state = state - def build_model(self, input_dim): + def build_model(self): with open(os.environ['PYNET_SAVE_PATH'] + '/log/' + self.state.hidden1.model + '/model.pkl') as f1: model = cPickle.load(f1) @@ -328,8 +360,12 @@ def run(self): dataset = self.build_dataset() learning_rule = self.build_learning_rule() - model = self.build_model(dataset.feature_size()) - model.layers[0].dropout_below = self.state.hidden1.dropout_below + model = self.build_model() + + if self.state.fine_tuning_only: + model.layers[0].dropout_below = None + # model.layers[0].blackout_below = None + print "Fine Tuning Only" if self.state.log.save_to_database_name: database = self.build_database(dataset, learning_rule, model) @@ -342,10 +378,13 @@ def run(self): model = model) train_obj.run() - log.info("fine tuning") - train_obj.model.layers[0].dropout_below = None - train_obj.setup() - train_obj.run() + + if not self.state.fine_tuning_only: + log.info("..Fine Tuning after Noisy Training") + train_obj.model.layers[0].dropout_below = None + # train_obj.model.layers[0].blackout_below = None + train_obj.setup() + train_obj.run() @@ -390,7 +429,7 @@ def run(self): learning_rule = learning_rule, model = model) - log.info("fine tuning") + log.info("Fine Tuning") train_obj.model.layers[0].dropout_below = None train_obj.model.layers[1].dropout_below = None train_obj.run() @@ -430,7 +469,16 @@ def run(self): learning_rule = self.build_learning_rule() model = self.build_model(dataset.feature_size()) - model.layers[0].dropout_below = self.state.hidden1.dropout_below + + if not self.state.fine_tuning_only: + model.layers[0].dropout_below = self.state.hidden1.dropout_below + # model.layers[0].blackout_below = self.state.hidden1.blackout_below + + model.layers[1].dropout_below = self.state.hidden2.dropout_below + # model.layers[1].blackout_below = self.state.hidden2.blackout_below + + model.layers[2].dropout_below = self.state.hidden3.dropout_below + # model.layers[2].blackout_below = self.state.hidden3.blackout_below if self.state.log.save_to_database_name: database = self.build_database(dataset, learning_rule, model) @@ -444,14 +492,15 @@ def run(self): learning_rule = learning_rule, model = model) - # train_obj.run() - - # fine tuning - log.info("fine tuning") - # train_obj.model.layers[0].dropout_below = None - # train_obj.setup() train_obj.run() + if not self.state.fine_tuning_only: + log.info("..Fine Tuning after Noisy Training") + train_obj.model.layers[0].dropout_below = None + # train_obj.model.layers[0].blackout_below = None + train_obj.setup() + train_obj.run() + class Laura_Two_Layers_No_Transpose(AE): diff --git a/hps/MLP.py b/hps/MLP.py new file mode 100644 index 0000000..e85744e --- /dev/null +++ b/hps/MLP.py @@ -0,0 +1,133 @@ + +from jobman import DD, expand, flatten + +import pynet.layer as layer +from pynet.model import * +from pynet.layer import * +from pynet.datasets.mnist import Mnist, Mnist_Blocks +import pynet.datasets.spec as spec +import pynet.datasets.mnist as mnist +import pynet.datasets.mapping as mapping +from pynet.learning_rule import LearningRule +from pynet.log import Log +from pynet.train_object import TrainObject +from pynet.cost import Cost +import pynet.datasets.preprocessor as preproc + +import cPickle +import os + +import theano +from theano.sandbox.cuda.var import CudaNdarraySharedVariable +floatX = theano.config.floatX + +class NN(): + + def __init__(self, state): + self.state = state + + def run(self): + dataset = self.build_dataset() + learning_rule = self.build_learning_rule() + model = self.build_model(dataset) + database = self.build_database(dataset, learning_rule, model) + log = self.build_log(database) + train_obj = TrainObject(log = log, + dataset = dataset, + learning_rule = learning_rule, + model = model) + train_obj.run() + + + def build_log(self, save_to_database=None, id=None): + log = Log(experiment_name = id is not None and '%s_%s'%(self.state.log.experiment_name,id) \ + or self.state.log.experiment_name, + description = self.state.log.description, + save_outputs = self.state.log.save_outputs, + save_learning_rule = self.state.log.save_learning_rule, + save_model = self.state.log.save_model, + save_epoch_error = self.state.log.save_epoch_error, + save_to_database = save_to_database) + return log + + + def build_dataset(self): + dataset = None + + preprocessor = None if self.state.dataset.preprocessor is None else \ + getattr(preproc, self.state.dataset.preprocessor)() + + if self.state.dataset.type == 'Mnist': + dataset = Mnist(train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, + preprocessor = preprocessor, + batch_size = self.state.dataset.batch_size, + num_batches = self.state.dataset.num_batches, + iter_class = self.state.dataset.iter_class, + rng = self.state.dataset.rng) + + + elif self.state.dataset.type[:12] == 'Mnist_Blocks': + dataset = getattr(mnist, self.state.dataset.type)( + feature_size = self.state.dataset.feature_size, + target_size = self.state.dataset.feature_size, + train_valid_test_ratio = self.state.dataset.train_valid_test_ratio, + preprocessor = preprocessor, + batch_size = self.state.dataset.batch_size, + num_batches = self.state.dataset.num_batches, + iter_class = self.state.dataset.iter_class, + rng = self.state.dataset.rng) + + return dataset + + + def build_learning_rule(self): + learning_rule = LearningRule(max_col_norm = self.state.learning_rule.max_col_norm, + learning_rate = self.state.learning_rule.learning_rate, + momentum = self.state.learning_rule.momentum, + momentum_type = self.state.learning_rule.momentum_type, + L1_lambda = self.state.learning_rule.L1_lambda, + L2_lambda = self.state.learning_rule.L2_lambda, + training_cost = Cost(type = self.state.learning_rule.cost), + stopping_criteria = {'max_epoch' : self.state.learning_rule.stopping_criteria.max_epoch, + 'epoch_look_back' : self.state.learning_rule.stopping_criteria.epoch_look_back, + 'cost' : Cost(type=self.state.learning_rule.stopping_criteria.cost), + 'percent_decrease' : self.state.learning_rule.stopping_criteria.percent_decrease}) + return learning_rule + + + def build_model(self, dataset): + model = MLP(input_dim=dataset.feature_size(), rand_seed=self.state.model.rand_seed) + + hidden1 = getattr(layer, self.state.hidden1.type)(dim=self.state.hidden1.dim, + name=self.state.hidden1.name, + dropout_below=self.state.hidden1.dropout_below, + blackout_below=self.state.hidden1.blackout_below) + model.add_layer(hidden1) + + output = getattr(layer, self.state.output.type)(dim=dataset.target_size(), + name=self.state.output.name, + dropout_below=self.state.output.dropout_below, + blackout_below=self.state.output.blackout_below) + model.add_layer(output) + return model + + + + def build_database(self, dataset, learning_rule, model): + save_to_database = {'name' : self.state.log.save_to_database_name, + 'records' : {'Dataset' : dataset.__class__.__name__, + 'max_col_norm' : learning_rule.max_col_norm, + 'Weight_Init_Seed' : model.rand_seed, + 'Dropout_Below' : str([layer.dropout_below for layer in model.layers]), + 'Blackout_Below' : str([layer.blackout_below for layer in model.layers]), + 'Batch_Size' : dataset.batch_size, + 'nblocks' : dataset.nblocks(), + 'Layer_Types' : str([layer.__class__.__name__ for layer in model.layers]), + 'Layer_Dim' : str([layer.dim for layer in model.layers]), + 'Preprocessor' : dataset.preprocessor.__class__.__name__, + 'Learning_Rate' : learning_rule.learning_rate, + 'Momentum' : learning_rule.momentum, + 'Training_Cost' : learning_rule.cost.type, + 'Stopping_Cost' : learning_rule.stopping_criteria['cost'].type} + } + return save_to_database diff --git a/hps/jobs/experiment.py b/hps/jobs/experiment.py index 3035579..e9d02ec 100644 --- a/hps/jobs/experiment.py +++ b/hps/jobs/experiment.py @@ -1,4 +1,5 @@ -from hps.models import * +from hps.AE import * +from hps.MLP import NN import os @@ -26,9 +27,14 @@ def setEnv(): print('PYNET_SAVE_PATH = ' + os.environ['PYNET_SAVE_PATH']) print('PYNET_DATABASE_PATH = ' + os.environ['PYNET_DATABASE_PATH']) +def NN_exp(state, channel): + setEnv() + hps = NN(state) + hps.run() -def AE_exp(state, channel): + return channel.COMPLETE +def AE_exp(state, channel): setEnv() hps = AE(state) hps.run() diff --git a/hps/launch.py b/hps/launch.py index f22644d..b748740 100644 --- a/hps/launch.py +++ b/hps/launch.py @@ -128,7 +128,7 @@ def get_cmd(model, mem, use_gpu, queue, host, duree, ppn, nb_proc, pmem, gpus): parser.add_argument('-n', '--total_number_jobs', type=int, dest='n_jobs', default=1, help='''The total number of jobs that will be launched on machines.''') - parser.add_argument('-m', '--mem', default='8000m', + parser.add_argument('-m', '--mem', default='10000m', help='''Memory usage limit by job in MB.''') parser.add_argument('-c', '--n_concur_jobs', type=int, @@ -149,7 +149,7 @@ def get_cmd(model, mem, use_gpu, queue, host, duree, ppn, nb_proc, pmem, gpus): parser.add_argument('--nb_proc', help='''number of jobs running at any one time''') - parser.add_argument('--pmem', default='8000m', help='''memory allocation per core''') + parser.add_argument('--pmem', default='10000m', help='''memory allocation per core''') parser.add_argument('--gpus', default='2', help='''memory allocation per core''') diff --git a/hps/model_config.py b/hps/model_config.py index 5285099..b6ec8f2 100644 --- a/hps/model_config.py +++ b/hps/model_config.py @@ -1,6 +1,95 @@ from jobman import DD, flatten model_config = DD({ + 'NN' : DD({ + + 'model' : DD({ + 'rand_seed' : None + }), # end mlp + + 'log' : DD({ + 'experiment_name' : 'MLP_Testing_Mnist_blackout', + 'description' : '', + 'save_outputs' : True, + 'save_learning_rule' : False, + 'save_model' : False, + 'save_epoch_error' : True, + 'save_to_database_name' : 'Testing2.db' + }), # end log + + + 'learning_rule' : DD({ + 'max_col_norm' : (1, 10, 50), + 'learning_rate' : (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.9), + 'momentum' : (1e-2, 1e-1, 0.5, 0.9), + 'momentum_type' : 'normal', + 'L1_lambda' : None, + 'L2_lambda' : None, + 'cost' : 'entropy', + 'stopping_criteria' : DD({ + 'max_epoch' : 100, + 'epoch_look_back' : 10, + 'cost' : 'error', + 'percent_decrease' : 0.05 + }) # end stopping_criteria + }), # end learning_rule + + 'dataset' : DD({ + + 'type' : 'Mnist', + 'train_valid_test_ratio': [8, 1, 1], + 'feature_size' : 784, + + # 'preprocessor' : None, + # 'preprocessor' : 'Scale', + # 'preprocessor' : 'GCN', + # 'preprocessor' : 'LogGCN', + # 'preprocessor' : 'Standardize', + + 'preprocessor' : DD({ + # 'type' : None, + 'type' : 'Scale', + # 'type' : 'GCN', + # 'type' : 'LogGCN', + # 'type' : 'Standardize', + + # for Scale + 'global_max' : 89, + 'global_min' : -23 + }), + + 'batch_size' : (50, 100, 150, 200), + 'num_batches' : None, + 'iter_class' : 'SequentialSubsetIterator', + 'rng' : None + }), # end dataset + + #============================[ Layers ]===========================# + 'hidden1' : DD({ + 'name' : 'hidden1', + 'type' : 'Sigmoid', + 'dim' : 500, + + # 'dropout_below' : (0.05, 0.1, 0.15, 0.2) + # 'dropout_below' : 0.5, + 'dropout_below' : None, + + 'blackout_below' : 0.5 + }), # end hidden_layer + + 'output' : DD({ + 'name' : 'output', + 'type' : 'Sigmoid', + + # 'dropout_below' : 0.5, + 'dropout_below' : None, + + 'blackout_below' : None + }) # end output_layer + }), # end autoencoder + + ############################[AE_Testing]########################## + ################################################################## 'AE_Testing' : DD({ @@ -13,8 +102,9 @@ 'experiment_name' : 'AE_Testing_Mnist_500_100', 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, + 'save_epoch_error' : True, 'save_to_database_name' : 'Database_Name.db' }), # end log @@ -83,8 +173,9 @@ 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, + 'save_epoch_error' : True, 'save_to_database_name' : 'Laura.db' }), # end log @@ -97,11 +188,11 @@ 'momentum_type' : 'normal', 'L1_lambda' : None, 'L2_lambda' : None, - 'cost' : 'mse', + 'cost' : 'entropy', 'stopping_criteria' : DD({ 'max_epoch' : 100, 'epoch_look_back' : 10, - 'cost' : 'mse', + 'cost' : 'entropy', 'percent_decrease' : 0.05 }) # end stopping_criteria }), # end learning_rule @@ -169,8 +260,8 @@ # 'experiment_name' : 'AE0901_Warp_Blocks_500_180_tanh_gpu', # 'experiment_name' : 'AE1016_Warp_Blocks_180_120_tanh_tanh_gpu_dropout', #helios - 'experiment_name' : 'AE1016_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_blackout', #helios - # + # 'experiment_name' : 'AE1018_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout', #helios + # 'experiment_name' : 'AE0919_Blocks_180_120_tanh_tanh_gpu_dropout', #helios # 'experiment_name' : 'AE0918_Blocks_180_120_tanh_tanh_gpu_clean', #helios @@ -178,7 +269,8 @@ # 'experiment_name' : 'AE0916_Blocks_180_120_tanh_tanh_gpu_output_sig_clean', # 'experiment_name' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_dropout', #helios - # 'experiment_name' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_clean', #helios + 'experiment_name' : 'AE1105_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_clean', #helios + 'description' : '', @@ -186,12 +278,12 @@ 'save_learning_rule' : True, 'save_model' : True, 'save_epoch_error' : True, - 'save_to_database_name' : 'Laura3.db' + 'save_to_database_name' : 'Laura5.db' }), # end log 'learning_rule' : DD({ - 'max_col_norm' : (1, 10, 50), + 'max_col_norm' : 1, # 'learning_rate' : ((1e-5, 0.5), float), 'learning_rate' : (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.5), 'momentum' : (1e-3, 1e-2, 1e-1, 0.5, 0.9), @@ -217,23 +309,38 @@ # 'type' : 'Laura_Blocks_500_Tanh_Sigmoid', # 'type' : 'Laura_Blocks_500', # 'type' : 'Laura_Blocks', - # 'type' : 'Laura_Warp_Blocks', + 'type' : 'Laura_Warp_Blocks', # 'type' : 'Laura_Warp_Standardize_Blocks', # 'type' : 'Laura_Standardize_Blocks', - 'type' : 'Laura_Scale_Warp_Blocks_500_Tanh', + # 'type' : 'Laura_Scale_Warp_Blocks_500_Tanh', # 'type' : 'Laura_Scale_Warp_Blocks_180_Tanh_Dropout', + # 'type' : 'Laura_Warp_Blocks_180_Tanh_Blackout', + # 'type' : 'Mnist', - 'feature_size' : 500, + 'feature_size' : 2049, 'train_valid_test_ratio': [8, 1, 1], - 'preprocessor' : None, - # 'preprocessor' : 'Scale', - # 'preprocessor' : 'GCN', - # 'preprocessor' : 'LogGCN', - # 'preprocessor' : 'Standardize', + 'dataset_noise' : DD({ + # 'type' : 'BlackOut', + # 'type' : 'MaskOut', + # 'type' : 'Gaussian', + 'type' : None + }), + + 'preprocessor' : DD({ + # 'type' : None, + 'type' : 'Scale', + # 'type' : 'GCN', + # 'type' : 'LogGCN', + # 'type' : 'Standardize', + + # for Scale + 'global_max' : 89, + 'global_min' : -23 + }), 'batch_size' : (50, 100, 150, 200), 'num_batches' : None, @@ -247,34 +354,39 @@ 'hidden1' : DD({ 'name' : 'hidden1', 'type' : 'Tanh', - 'dim' : 120, + 'dim' : 500, 'dropout_below' : None, # 'dropout_below' : (0.3, 0.4, 0.5), # 'dropout_below' : 0.5, # 'blackout_below' : None, - 'blackout_below' : 0.5 - - }), # end hidden_layer + # 'blackout_below' : 0.5 - 'hidden2' : DD({ - 'name' : 'hidden2', - 'type' : 'RELU', - 'dim' : 100, - 'dropout_below' : None, + 'layer_noise' : None, + # 'layer_noise' : 'BlackOut', + # 'layer_noise' : 'Gaussian', + # 'layer_noise' : 'MaskOut', - 'blackout_below' : None }), # end hidden_layer - 'h2_mirror' : DD({ - 'name' : 'h2_mirror', - 'type' : 'RELU', - # 'dim' : 2049, # dim = input.dim - 'dropout_below' : None, - - 'blackout_below' : None - }), # end output_layer + # 'hidden2' : DD({ + # 'name' : 'hidden2', + # 'type' : 'RELU', + # 'dim' : 100, + # 'dropout_below' : None, + # + # 'blackout_below' : None + # }), # end hidden_layer + # + # 'h2_mirror' : DD({ + # 'name' : 'h2_mirror', + # 'type' : 'RELU', + # # 'dim' : 2049, # dim = input.dim + # 'dropout_below' : None, + # + # 'blackout_below' : None + # }), # end output_layer 'h1_mirror' : DD({ 'name' : 'h1_mirror', @@ -284,36 +396,39 @@ 'dropout_below' : None, # 'dropout_below' : 0.5, - 'blackout_below' : None + # 'blackout_below' : None }) # end output_layer }), # end autoencoder - ########################[Laura_Continue]######################## + ##########################[Laura_Continue]######################## ################################################################## 'Laura_Continue' : DD({ + + 'model' : DD({ - 'rand_seed' : 252 + 'rand_seed' : 5343 }), # end mlp 'log' : DD({ - 'experiment_name' : 'AE1003_Scale_Warp_Blocks_3Layers_finetune_2049_120_tanh_tanh_gpu_noisy', + 'experiment_name' : 'AE1029_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout_continue', 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, - 'save_to_database_name' : 'Laura2.db' + 'save_epoch_error' : True, + 'save_to_database_name' : 'Laura4.db' }), # end log 'learning_rule' : DD({ - 'max_col_norm' : 50, - 'learning_rate' : 0.002109157240622, + 'max_col_norm' : 1, + 'learning_rate' : 0.001, # 'learning_rate' : ((1e-5, 9e-1), float), # 'learning_rate' : 0.01, 'momentum' : 0.01, @@ -337,28 +452,33 @@ # 'type' : 'Laura_Blocks', 'type' : 'Laura_Warp_Blocks', # 'type' : 'Mnist_Blocks', + # 'type' : 'Laura_Scale_Warp_Blocks_500_Tanh', 'feature_size' : 2049, 'train_valid_test_ratio': [8, 1, 1], - # 'preprocessor' : None, - 'preprocessor' : 'Scale', - # 'preprocessor' : 'GCN', - # 'preprocessor' : 'LogGCN', - # 'preprocessor' : 'Standardize', + 'preprocessor' : DD({ + # 'type' : None, + 'type' : 'Scale', + # 'type' : 'GCN', + # 'type' : 'LogGCN', + # 'type' : 'Standardize', + + # for Scale + 'global_max' : 89, + 'global_min' : -23 + }), - 'batch_size' : 50, + 'batch_size' : 150, 'num_batches' : None, 'iter_class' : 'SequentialSubsetIterator', 'rng' : None }), # end dataset # #============================[ Layers ]===========================# - + 'fine_tuning_only' : True, 'hidden1' : DD({ 'name' : 'hidden1', - 'model' : 'AE1002_Scale_Warp_Blocks_3Layers_finetune_2049_120_tanh_tanh_gpu_noisy_20141003_0046_17962047', - - 'dropout_below' : None, + 'model' : 'AE1028_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout_continue_20141028_2013_56321135', }), # end hidden_layer @@ -373,19 +493,20 @@ }), # end mlp 'log' : DD({ - 'experiment_name' : 'AE1004_Scale_Warp_Blocks_2Layers_finetune_2049_120_tanh_tanh_gpu_clean', + 'experiment_name' : 'AE1028_Scale_Warp_Blocks_2Layers_finetune_2049_180_tanh_tanh_gpu_blackout', 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, - 'save_to_database_name' : 'Laura2.db' + 'save_epoch_error' : True, + 'save_to_database_name' : 'Laura3.db' }), # end log 'learning_rule' : DD({ # 'max_col_norm' : (1, 10, 50), - 'max_col_norm' : 50, + 'max_col_norm' : 1, 'learning_rate' : (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.5), # 'learning_rate' : ((1e-5, 9e-1), float), # 'learning_rate' : 0.01, @@ -412,11 +533,18 @@ 'feature_size' : 2049, 'train_valid_test_ratio': [8, 1, 1], - # 'preprocessor' : None, - 'preprocessor' : 'Scale', - # 'preprocessor' : 'GCN', - # 'preprocessor' : 'LogGCN', - # 'preprocessor' : 'Standardize', + 'preprocessor' : DD({ + # 'type' : None, + # 'type' : 'Scale', + 'type' : 'GCN', + # 'type' : 'LogGCN', + # 'type' : 'Standardize', + + # for Scale + 'global_max' : 89, + 'global_min' : -23 + }), + 'batch_size' : (50, 100, 150, 200), 'num_batches' : None, 'iter_class' : 'SequentialSubsetIterator', @@ -429,7 +557,7 @@ 'name' : 'hidden1', # 'model' : 'AE0911_Warp_Blocks_2049_500_tanh_tanh_gpu_clean_20140912_2337_04263067', - 'model' : 'AE0930_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_clean_20140930_1345_29800576', + 'model' : 'AE1018_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout_continue_20141018_1408_44747438', 'dropout_below' : None, # 'dropout_below' : (0.1, 0.2, 0.3, 0.4, 0.5), # 'dropout_below' : 0.1, @@ -439,15 +567,17 @@ 'name' : 'hidden2', # 'model' : 'AE1001_Warp_Blocks_500_120_tanh_tanh_gpu_clean_20141003_0113_02206401', - 'model' : 'AE1003_Scale_Warp_Blocks_500_120_tanh_tanh_gpu_clean_20141004_0640_00423949', + 'model' : 'AE1018_Warp_Blocks_500_180_tanh_tanh_gpu_blackout_20141018_2238_56949300', 'dropout_below' : None, }) }), # end autoencoder ########################[Laura_Three_Layers]######################## - ################################################################## + #################################################################### 'Laura_Three_Layers' : DD({ + 'fine_tuning_only' : False, + 'model' : DD({ 'rand_seed' : None }), # end mlp @@ -463,23 +593,25 @@ # 'experiment_name' : 'AE0917_Warp_Blocks_3layers_finetune_2049_120_tanh_tanh_gpu_clean', # 'experiment_name' : 'AE0919_Warp_Blocks_3layers_finetune_2049_120_tanh_tanh_gpu_noisy', - 'experiment_name' : 'AE1002_Scale_Warp_Blocks_3Layers_finetune_2049_120_tanh_tanh_gpu_noisy', + # 'experiment_name' : 'AE1002_Scale_Warp_Blocks_3Layers_finetune_2049_120_tanh_tanh_gpu_noisy', # 'experiment_name' : 'AE1002_Scale_Warp_Blocks_3Layers_finetune_2049_120_tanh_tanh_gpu_clean', + 'experiment_name' : 'AE1019_Warp_Blocks_3layers_finetune_2049_120_tanh_tanh_gpu_blackout', 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, - 'save_to_database_name' : 'Laura2.db' + 'save_epoch_error' : True, + 'save_to_database_name' : 'Laura3.db' }), # end log 'learning_rule' : DD({ - 'max_col_norm' : (1, 10, 50), + 'max_col_norm' : 1, # 'max_col_norm' : 50, - # 'learning_rate' : (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.5), - 'learning_rate' : ((1e-5, 9e-1), float), + 'learning_rate' : (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.5), + # 'learning_rate' : ((1e-5, 9e-1), float), # 'learning_rate' : 0.01, 'momentum' : (1e-3, 1e-2, 1e-1, 0.5, 0.9), # 'momentum' : 0.05, @@ -505,8 +637,8 @@ 'train_valid_test_ratio': [8, 1, 1], # 'preprocessor' : None, - 'preprocessor' : 'Scale', - # 'preprocessor' : 'GCN', + # 'preprocessor' : 'Scale', + 'preprocessor' : 'GCN', # 'preprocessor' : 'LogGCN', # 'preprocessor' : 'Standardize', 'batch_size' : (50, 100, 150, 200), @@ -525,12 +657,16 @@ # 'model' :'AE0912_Blocks_2049_500_tanh_tanh_gpu_clean_20140914_1242_27372903', # 'model' : 'AE0915_Blocks_2049_500_tanh_tanh_gpu_Dropout_20140915_1900_37160748', - 'model' : 'AE1002_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_dropout_20141001_0321_33382955', + # 'model' : 'AE1002_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_dropout_20141001_0321_33382955', # 'model' : 'AE0930_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_clean_20140930_1345_29800576', + 'model' : 'AE1018_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout_continue_20141018_1408_44747438', + 'dropout_below' : None, # 'dropout_below' : (0.1, 0.2, 0.3, 0.4, 0.5), # 'dropout_below' : 0.1, + + 'blackout_below' : 0.5 }), # end hidden_layer 'hidden2' : DD({ @@ -541,10 +677,14 @@ # 'model' : 'AE0916_Blocks_500_180_tanh_tanh_gpu_clean_20140916_2255_06553688', # 'model' : 'AE0918_Blocks_500_180_tanh_tanh_gpu_dropout_20140918_0920_42738052', - 'model' : 'AE1001_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_dropout_20141001_2158_16765065', + # 'model' : 'AE1001_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_dropout_20141001_2158_16765065', # 'model' : 'AE1001_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_clean_20141002_0348_53679208', + 'model' : 'AE1018_Warp_Blocks_500_180_tanh_tanh_gpu_blackout_20141018_2238_56949300', + 'dropout_below' : None, + + 'blackout_below' : None }), # end hidden_layer 'hidden3' : DD({ @@ -556,10 +696,14 @@ # 'model' : 'AE0919_Blocks_180_120_tanh_tanh_gpu_dropout_20140919_1345_22865393', # 'model' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_dropout_20141002_1711_48207269', - 'model' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_dropout_20141002_1457_08966968', + # 'model' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_dropout_20141002_1457_08966968', # 'model' : 'AE1001_Scale_Warp_Blocks_180_120_tanh_tanh_gpu_clean_20141002_1713_16791523', + 'model' : 'AE1018_Warp_Blocks_180_120_tanh_tanh_gpu_blackout_20141019_1322_55969785', + 'dropout_below' : None, + + 'blackout_below' : None }), # end hidden_layer @@ -578,8 +722,9 @@ 'experiment_name' : 'AE0730_No_Transpose_Warp_Blocks_180_64', 'description' : '', 'save_outputs' : True, - 'save_learning_rule' : True, + 'save_learning_rule' : True, 'save_model' : True, + 'save_epoch_error' : True, 'save_to_database_name' : 'Laura.db' }), # end log diff --git a/pynet/datasets/cifar10.py b/pynet/datasets/cifar10.py index 40730c3..d6d7c91 100644 --- a/pynet/datasets/cifar10.py +++ b/pynet/datasets/cifar10.py @@ -8,9 +8,9 @@ from pynet.utils.mnist_ubyte import read_mnist_images from pynet.utils.mnist_ubyte import read_mnist_labels -from pynet.datasets.dataset import IterMatrix, Dataset, DataBlocks +from pynet.datasets.dataset import SingleBlock -class Cifar10(Dataset): +class Cifar10(SingleBlock): def __init__(self, **kwargs): diff --git a/pynet/datasets/dataset.py b/pynet/datasets/dataset.py index 0eb2300..19f37a4 100644 --- a/pynet/datasets/dataset.py +++ b/pynet/datasets/dataset.py @@ -1,72 +1,12 @@ -import logging -logger = logging.getLogger(__name__) -import pynet.datasets.iterator as iter - - -class SingleBlock(object): - ''' - Interface for Dataset with single block - ''' - - def __iter__(self): - self.iter = True - return self - def next(self): - if self.iter: - self.iter = False - return self - else: - raise StopIteration - - def nblocks(self): - return 1 - - -class DataBlocks(object): - ''' - Interface for large dataset broken up into many blocks and train one block - of Dataset at a time - ''' - - def __init__(self, feature_size, target_size, train_valid_test_ratio = [8,1,1], - preprocessor=None, batch_size=100, num_batches=None, - iter_class='SequentialSubsetIterator', rng=None): - - self.featureSize = feature_size - self.targetSize = target_size - - self.ratio = train_valid_test_ratio - self.preprocessor = preprocessor - self.iter_class = iter_class - self.batch_size = batch_size - self.num_batches = num_batches - self.rng = rng +import pynet.datasets.iterator as iterators +import numpy as np - assert len(train_valid_test_ratio) == 3, 'the size of list is not 3' - self.dataset = Dataset(X=None, y=None, - train_valid_test_ratio=train_valid_test_ratio, - preprocessor=preprocessor, batch_size=batch_size, - num_batches=num_batches, iter_class=iter_class, rng=rng) - - def __iter__(self): - raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") - - def next(self): - ''' - Loads the next Dataset() from the disk - ''' - raise NotImplementedError(str(type(self))+" does not implement the next method.") - - def nblocks(self): - raise NotImplementedError(str(type(self))+" does not implement the nblocks method.") - - def feature_size(self): - return self.featureSize - - def target_size(self): - return self.targetSize +import logging +internal_logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) +from pynet.log import Log class IterMatrix(object): @@ -79,7 +19,7 @@ def __init__(self, X, y, iter_class='SequentialSubsetIterator', self.num_batches = num_batches self.iter_class = iter_class self.rng = rng - self.iterator = getattr(iter, self.iter_class) + self.iterator = getattr(iterators, self.iter_class) def __iter__(self): return self.iterator(dataset_size=self.dataset_size(), @@ -99,40 +39,89 @@ def feature_size(self): def target_size(self): return self.y.shape[1] -class Dataset(SingleBlock): - def __init__(self, X=None, y=None, train_valid_test_ratio = [8,1,1], - preprocessor=None, batch_size=100, num_batches=None, - iter_class='SequentialSubsetIterator', rng=None): +class Dataset(object): + + def __init__(self, train_valid_test_ratio = [8,1,1], + preprocessor=None, noise=None, batch_size=100, num_batches=None, + iter_class='SequentialSubsetIterator', rng=None, log=None): ''' - DESCRIPTION: Interface that contains three IterMatrix - PARAM: - X : 2d numpy of size [num_examples, num_features] - y : 2d numpy of size [num_examples, num_targets] - train_valid_test_ratio : list - the ratio to split the dataset linearly + DESCRIPTION: Abstract class ''' assert len(train_valid_test_ratio) == 3, 'the size of list is not 3' self.ratio = train_valid_test_ratio self.preprocessor = preprocessor + self.noise = noise self.iter_class = iter_class self.batch_size = batch_size self.num_batches = num_batches self.rng = rng + self.log = log + + if self.log is None: + # use default Log setting + self.log = Log(logger=internal_logger) + + def __iter__(self): + raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") + + def next(self): + raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") + + def nblocks(self): + raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") + + def feature_size(self): + raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") + + def target_size(self): + raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") + + +class SingleBlock(Dataset): + + def __init__(self, X=None, y=None, **kwargs): + super(SingleBlock, self).__init__(**kwargs) + self.train = None self.valid = None self.test = None - if X is not None and y is not None: + assert len(self.ratio) == 3, 'the size of list is not 3' + + if X and y: assert X.shape[0] == y.shape[0], 'the number of examples in input and target dont match' - if self.preprocessor is not None: - logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + if self.preprocessor: + self.log.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) X = self.preprocessor.apply(X) + if self.noise: + self.log.info('..applying noise: ' + self.noise.__class__.__name__) + X = self.noise.apply(X) + self.set_Xy(X, y) + def __iter__(self): + self.iter = True + return self + + def next(self): + if self.iter: + self.iter = False + return self + else: + raise StopIteration + + def nblocks(self): + return 1 + + def feature_size(self): + return self.train.X.shape[1] + + def target_size(self): + return self.train.y.shape[1] def set_Xy(self, X, y): num_examples = X.shape[0] @@ -150,21 +139,21 @@ def set_Xy(self, X, y): test_y = y[num_train+num_valid:] if self.ratio[0] == 0: - logger.warning('Train set is empty!') + self.log.warning('Train set is empty!') self.train = IterMatrix(train_X, train_y, iter_class=self.iter_class, batch_size=self.batch_size, num_batches=self.num_batches, rng=self.rng) if self.ratio[1] == 0: - logger.warning('Valid set is empty! It is needed for stopping of training') + self.log.warning('Valid set is empty! It is needed for stopping of training') self.valid = IterMatrix(valid_X, valid_y, iter_class=self.iter_class, batch_size=self.batch_size, num_batches=self.num_batches, rng=self.rng) if self.ratio[2] == 0: - logger.warning('Test set is empty! It is needed for saving the best model') + self.log.warning('Test set is empty! It is needed for saving the best model') self.test = IterMatrix(test_X, test_y, iter_class=self.iter_class, batch_size=self.batch_size, @@ -192,8 +181,258 @@ def set_test(self, X, y): self.test.X = X self.test.y = y + +class DataBlocks(Dataset): + + def __init__(self, feature_size, target_size, data_paths, **kwargs): + + """ + DESCRIPTION: + This is class for processing blocks of data. + PARAM: + data_paths(list): contains the paths to the numpy data files. It's a + list of tuples whereby the first element of the tuple + is the X path, and the second is the y path. + """ + super(DataBlocks, self).__init__(**kwargs) + self.featureSize = feature_size + self.targetSize = target_size + assert isinstance(data_paths, list), "data_paths is not a list" + self.data_paths = data_paths + self.single_block = SingleBlock(X=None, y=None, **kwargs) + + def __iter__(self): + self.files = iter(self.data_paths) + return self + + def next(self): + file = next(self.files) + # if isinstance(file, str): + # with open(file, 'rb') as f: + # data = np.load(f) + # X = data + # + # if self.preprocessor: + # self.log.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + # data = self.preprocessor.apply(data) + # X = data + # + # if self.noise: + # self.log.info('..applying noise: ' + self.noise.__class__.__name__) + # noisy_data = self.noise.apply(data) + # X = noisy_data + # + # self.single_block.set_Xy(X=X, y=data) + # return self.single_block + + assert isinstance(file, tuple), str(type(file)) + "is not a tuple" + with open(file[0], 'rb') as X_fin, open(file[1], 'rb') as y_fin: + data = np.load(X_fin) + X = data + y = np.load(y_fin) + + if self.preprocessor: + self.log.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + data = self.preprocessor.apply(data) + X = data + + if self.noise: + self.log.info('..applying noise: ' + self.noise.__class__.__name__) + noisy_data = self.noise.apply(data) + X = noisy_data + + self.single_block.set_Xy(X=X, y=y) + return self.single_block + + + + + def nblocks(self): + return len(self.data_paths) + def feature_size(self): - return self.train.X.shape[1] + return self.featureSize def target_size(self): - return self.train.y.shape[1] + return self.targetSize + + + + + ##################################### +# class SingleBlock(object): +# ''' +# Interface for Dataset with single block +# ''' +# +# def __iter__(self): +# self.iter = True +# return self +# +# def next(self): +# if self.iter: +# self.iter = False +# return self +# else: +# raise StopIteration +# +# def nblocks(self): +# return 1 +# +# +# class SingleBlock(Dataset): +# +# def __init__(self, X=None, y=None, train_valid_test_ratio = [8,1,1], +# preprocessor=None, noise=None, batch_size=100, num_batches=None, +# iter_class='SequentialSubsetIterator', rng=None): +# +# ''' +# DESCRIPTION: Interface that contains three IterMatrix +# PARAM: +# X : 2d numpy of size [num_examples, num_features] +# y : 2d numpy of size [num_examples, num_targets] +# train_valid_test_ratio : list +# the ratio to split the dataset linearly +# ''' +# +# assert len(train_valid_test_ratio) == 3, 'the size of list is not 3' +# self.ratio = train_valid_test_ratio +# self.preprocessor = preprocessor +# self.noise = noise +# self.iter_class = iter_class +# self.batch_size = batch_size +# self.num_batches = num_batches +# self.rng = rng +# +# self.train = None +# self.valid = None +# self.test = None +# +# assert len(train_valid_test_ratio) == 3, 'the size of list is not 3' +# +# if X is not None and y is not None: +# assert X.shape[0] == y.shape[0], 'the number of examples in input and target dont match' +# if self.preprocessor: +# self.log.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) +# X = self.preprocessor.apply(X) +# if self.noise: +# self.log.info('..applying noise: ' + self.noise.__class__.__name__) +# X = self.noise.apply(X) +# +# self.set_Xy(X, y) +# +# +# def set_Xy(self, X, y): +# num_examples = X.shape[0] +# total_ratio = sum(self.ratio) +# num_train = int(self.ratio[0] * 1.0 * num_examples / total_ratio) +# num_valid = int(self.ratio[1] * 1.0 * num_examples / total_ratio) +# +# train_X = X[:num_train] +# train_y = y[:num_train] +# +# valid_X = X[num_train:num_train+num_valid] +# valid_y = y[num_train:num_train+num_valid] +# +# test_X = X[num_train+num_valid:] +# test_y = y[num_train+num_valid:] +# +# if self.ratio[0] == 0: +# self.log.warning('Train set is empty!') +# +# self.train = IterMatrix(train_X, train_y, iter_class=self.iter_class, +# batch_size=self.batch_size, +# num_batches=self.num_batches, rng=self.rng) +# +# if self.ratio[1] == 0: +# self.log.warning('Valid set is empty! It is needed for stopping of training') +# +# self.valid = IterMatrix(valid_X, valid_y, iter_class=self.iter_class, +# batch_size=self.batch_size, +# num_batches=self.num_batches, rng=self.rng) +# +# if self.ratio[2] == 0: +# self.log.warning('Test set is empty! It is needed for saving the best model') +# +# self.test = IterMatrix(test_X, test_y, iter_class=self.iter_class, +# batch_size=self.batch_size, +# num_batches=self.num_batches, rng=self.rng) +# +# +# def get_train(self): +# return self.train +# +# def get_valid(self): +# return self.valid +# +# def get_test(self): +# return self.test +# +# def set_train(self, X, y): +# self.train.X = X +# self.train.y = y +# +# def set_valid(self, X, y): +# self.valid.X = X +# self.valid.y = y +# +# def set_test(self, X, y): +# self.test.X = X +# self.test.y = y +# +# def feature_size(self): +# return self.train.X.shape[1] +# +# def target_size(self): +# return self.train.y.shape[1] +# +# +# class DataBlocks(Dataset): +# ''' +# Interface for large dataset broken up into many blocks and train one block +# of Dataset at a time +# ''' +# +# # def __init__(self, feature_size, target_size, train_valid_test_ratio = [8,1,1], +# # preprocessor=None, noise=None, batch_size=100, num_batches=None, +# # iter_class='SequentialSubsetIterator', rng=None): +# # +# # self.featureSize = feature_size +# # self.targetSize = target_size +# # +# # self.ratio = train_valid_test_ratio +# # self.preprocessor = preprocessor +# # self.noise = noise +# # self.iter_class = iter_class +# # self.batch_size = batch_size +# # self.num_batches = num_batches +# # self.rng = rng +# # +# # assert len(train_valid_test_ratio) == 3, 'the size of list is not 3' +# # self.dataset = Dataset(X=None, y=None, +# # train_valid_test_ratio=train_valid_test_ratio, +# # preprocessor=preprocessor, noise=noise, batch_size=batch_size, +# # num_batches=num_batches, iter_class=iter_class, rng=rng) +# +# def __init__(feature_size, target_size, **kwargs): +# self.feature_size = feature_size +# self.target_size = target_size +# super(DataBlocks, self).__init__(**kwargs) +# +# def __iter__(self): +# raise NotImplementedError(str(type(self))+" does not implement the __iter__ method.") +# +# def next(self): +# ''' +# Loads the next Dataset() from the disk +# ''' +# raise NotImplementedError(str(type(self))+" does not implement the next method.") +# +# def nblocks(self): +# raise NotImplementedError(str(type(self))+" does not implement the nblocks method.") +# +# def feature_size(self): +# return self.featureSize +# +# def target_size(self): +# return self.targetSize diff --git a/pynet/datasets/dataset_noise.py b/pynet/datasets/dataset_noise.py new file mode 100644 index 0000000..9c6785e --- /dev/null +++ b/pynet/datasets/dataset_noise.py @@ -0,0 +1,71 @@ + + +""" +Functionality : Define the noise that is to be added to the dataset +""" + +import numpy as np + +class Noise(object): + """ + This is an abstract class for applying noise to dataset + """ + + def apply(self, X): + """ + DESCRIPTION: + This method applies noise to X and return a noisy X + PARAM: + X : 2d numpy array of dimension number of examples by number of dimensions + """ + raise NotImplementedError(str(type(self))+" does not implement an apply method.") + + + +class MaskOut(Noise): + + """ + This noise masked out a portion of the dimension from each example + """ + + def __init__(self, ratio=0.5): + """ + PARAM: + ratio : float + The portion of the inputs that is masked out + """ + self.ratio = ratio + + def apply(self, X): + return X * np.random.binomial(size=X.shape, n=1, p=(1-self.ratio)) + + +class Gaussian(Noise): + """ + Applies gaussian noise to each value of X + """ + + def __init__(self, std=0.1, mean=0): + self.std = std + self.mean = mean + + def apply(self, X): + return X + np.random.normal(loc=self.mean, scale=self.std, size=X.shape) + + +class BlackOut(Noise): + """ + This noise masked out a random example in a dataset, + adding noise in the time dimension + """ + + def __init__(self, ratio=0.5): + """ + PARAM: + ratio : float + The portion of the examples that is masked out + """ + self.ratio = ratio + + def apply(self, X): + return X * np.random.binomial(size=(X.shape[0],1), n=1, p=(1-self.ratio)) diff --git a/pynet/datasets/mnist.py b/pynet/datasets/mnist.py index d9f3350..ffbb72a 100644 --- a/pynet/datasets/mnist.py +++ b/pynet/datasets/mnist.py @@ -6,9 +6,9 @@ from pynet.utils.mnist_ubyte import read_mnist_images from pynet.utils.mnist_ubyte import read_mnist_labels -from pynet.datasets.dataset import IterMatrix, Dataset, DataBlocks +from pynet.datasets.dataset import SingleBlock, DataBlocks -class Mnist(Dataset): +class Mnist(SingleBlock): def __init__(self, **kwargs): @@ -43,33 +43,36 @@ def __init__(self, **kwargs): class Mnist_Blocks(DataBlocks): - def __init__(self, feature_size, target_size, **kwargs): + def __init__(self, feature_size=784, target_size=10, **kwargs): # self.parts = [ 'blk1.npy', 'blk2.npy'] - self.parts = ['fullblk.npy'] - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/mnist_npy' - super(Mnist_Blocks, self).__init__(feature_size, target_size, **kwargs) - - - def __iter__(self): - self.files = iter(self.parts) - return self - - def next(self): - self.dataset.train = None - self.dataset.valid = None - self.dataset.test = None - with open(self.data_dir + '/' + next(self.files), 'rb') as f: - data = np.load(f) - if self.dataset.preprocessor is not None: - logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) - data = self.dataset.preprocessor.apply(data) - self.dataset.set_Xy(X=data, y=data) - data = None - return self.dataset - - def nblocks(self): - return len(self.parts) + parts = ['fullblk.npy'] + data_dir = os.environ['PYNET_DATA_PATH'] + '/mnist_npy' + data_paths = ["%s/%s"%(data_dir, part) for part in parts] + super(Mnist_Blocks, self).__init__(data_paths=data_paths, + feature_size=feature_size, + target_size=target_size, **kwargs) + + + # def __iter__(self): + # self.files = iter(self.parts) + # return self + # + # def next(self): + # self.dataset.train = None + # self.dataset.valid = None + # self.dataset.test = None + # with open(self.data_dir + '/' + next(self.files), 'rb') as f: + # data = np.load(f) + # if self.dataset.preprocessor is not None: + # logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + # data = self.dataset.preprocessor.apply(data) + # self.dataset.set_Xy(X=data, y=data) + # data = None + # return self.dataset + # + # def nblocks(self): + # return len(self.parts) class Mnist_Blocks_500(Mnist_Blocks): def __init__(self, feature_size, target_size, **kwargs): diff --git a/pynet/datasets/preprocessor.py b/pynet/datasets/preprocessor.py index f328cfc..ba0b411 100644 --- a/pynet/datasets/preprocessor.py +++ b/pynet/datasets/preprocessor.py @@ -283,20 +283,18 @@ class Scale(Preprocessor): """ - def __init__(self, global_max=None, global_min=None, use_local=True, scale_range=[-1,1], buffer=0.5): + def __init__(self, global_max=None, global_min=None, scale_range=[-1,1], buffer=0.5): self.scale_range = scale_range self.buffer = buffer self.max = global_max self.min = global_min - self.use_local = use_local assert scale_range[0] + buffer < scale_range[1] - buffer, \ 'the lower bound is larger than the upper bound' def apply(self, X): - # TODO allow local scaling of each example - self.max = self.max if self.max is not None else X.max() - self.min = self.min if self.min is not None else X.min() + self.max = self.max if self.max else X.max() + self.min = self.min if self.min else X.min() width = self.max - self.min assert width > 0, 'the max is not bigger than the min' scale = (self.scale_range[1] - self.scale_range[0] - 2 * self.buffer) / width diff --git a/pynet/datasets/spec.py b/pynet/datasets/spec.py index a2e8576..830b532 100644 --- a/pynet/datasets/spec.py +++ b/pynet/datasets/spec.py @@ -3,13 +3,13 @@ import os import numpy as np import theano -from pynet.datasets.dataset import Dataset, DataBlocks +from pynet.datasets.dataset import Dataset, DataBlocks, SingleBlock import glob import gc from pynet.utils.check_memory import print_mem_usage -class P276(Dataset): +class P276(SingleBlock): def __init__(self, **kwargs): @@ -19,7 +19,7 @@ def __init__(self, **kwargs): super(P276, self).__init__(X=data, y=data, **kwargs) -class Laura_Test(Dataset): +class Laura_Test(SingleBlock): def __init__(self, **kwargs): data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_npy' @@ -28,64 +28,141 @@ def __init__(self, **kwargs): super(Laura_Test, self).__init__(X=data, y=data, **kwargs) -class Laura_Blocks(DataBlocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks, self).__init__(feature_size, target_size, **kwargs) - self.parts = [ 'Laura_data_000.npy', 'Laura_data_010.npy', - 'Laura_data_001.npy', 'Laura_data_011.npy', - 'Laura_data_002.npy', 'Laura_data_012.npy', - 'Laura_data_003.npy', 'Laura_data_013.npy', - 'Laura_data_004.npy', 'Laura_data_014.npy', - 'Laura_data_005.npy', 'Laura_data_015.npy', - 'Laura_data_006.npy', 'Laura_data_016.npy', - 'Laura_data_007.npy', 'Laura_data_017.npy', - 'Laura_data_008.npy', 'Laura_data_018.npy', - 'Laura_data_009.npy', 'Laura_data_019.npy'] - - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_npy' +class Laura_Root(DataBlocks): + def __init__(self, data_paths, **kwargs): + super(Laura_Root, self).__init__(data_paths=data_paths, **kwargs) def __iter__(self): - self.files = iter(self.parts) + self.files = iter(self.data_paths) return self def next(self): - self.dataset.train = None - self.dataset.valid = None - self.dataset.test = None - with open(self.data_dir + '/' + next(self.files), 'rb') as f: + file = next(self.files) + with open(file, 'rb') as f: data = np.load(f) - if self.dataset.preprocessor is not None: + # X = data + + if self.preprocessor and self.noise: + logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + proc_data = self.preprocessor.apply(data) + data = None + logger.info('..applying noise: ' + self.noise.__class__.__name__) + noisy_data = self.noise.apply(proc_data) + + self.single_block.set_Xy(X=noisy_data, y=proc_data) + return self.single_block + + # X = data + + if self.noise: + logger.info('..applying noise: ' + self.noise.__class__.__name__) + noisy_data = self.noise.apply(data) + # X = noisy_data + + self.single_block.set_Xy(X=noisy_data, y=data) + return self.single_block + + if self.preprocessor: logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) - data = self.dataset.preprocessor.apply(data) - self.dataset.set_Xy(X=data, y=data) - data = None - return self.dataset - - def nblocks(self): - return len(self.parts) - -class Laura_Warp_Blocks(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks, self).__init__(feature_size, target_size, **kwargs) - self.parts = [ 'Laura_warp_data_000.npy', 'Laura_warp_data_010.npy', - 'Laura_warp_data_001.npy', 'Laura_warp_data_011.npy', - 'Laura_warp_data_002.npy', 'Laura_warp_data_012.npy', - 'Laura_warp_data_003.npy', 'Laura_warp_data_013.npy', - 'Laura_warp_data_004.npy', 'Laura_warp_data_014.npy', - 'Laura_warp_data_005.npy', 'Laura_warp_data_015.npy', - 'Laura_warp_data_006.npy', 'Laura_warp_data_016.npy', - 'Laura_warp_data_007.npy', 'Laura_warp_data_017.npy', - 'Laura_warp_data_008.npy', 'Laura_warp_data_018.npy', - 'Laura_warp_data_009.npy', 'Laura_warp_data_019.npy'] - - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_warp_npy' - -class Laura_Warp_Standardize_Blocks(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Standardize_Blocks, self).__init__(feature_size, target_size, **kwargs) - self.parts = [ 'Laura_warp_standardize_data_000.npy', 'Laura_warp_standardize_data_010.npy', + proc_data = self.preprocessor.apply(data) + data = None + self.single_block.set_Xy(X=proc_data, y=proc_data) + return self.single_block + + + + self.single_block.set_Xy(X=data, y=data) + return self.single_block + + + + + +class Laura_Blocks(Laura_Root): + + def __init__(self, data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_npy', **kwargs): + parts = [ 'Laura_data_000.npy', 'Laura_data_010.npy', + 'Laura_data_001.npy', 'Laura_data_011.npy', + 'Laura_data_002.npy', 'Laura_data_012.npy', + 'Laura_data_003.npy', 'Laura_data_013.npy', + 'Laura_data_004.npy', 'Laura_data_014.npy', + 'Laura_data_005.npy', 'Laura_data_015.npy', + 'Laura_data_006.npy', 'Laura_data_016.npy', + 'Laura_data_007.npy', 'Laura_data_017.npy', + 'Laura_data_008.npy', 'Laura_data_018.npy', + 'Laura_data_009.npy', 'Laura_data_019.npy'] + data_paths = ["%s/%s"%(data_dir, part) for part in parts] + super(Laura_Blocks, self).__init__(data_paths=data_paths, **kwargs) + + + # def __iter__(self): + # self.files = iter(self.data_paths) + # return self + # + # def next(self): + # file = next(self.files) + # with open(file, 'rb') as f: + # data = np.load(f) + # X = data + # + # if self.preprocessor: + # logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + # data = self.preprocessor.apply(data) + # X = data + # + # if self.noise: + # logger.info('..applying noise: ' + self.noise.__class__.__name__) + # noisy_data = self.noise.apply(data) + # X = noisy_data + # + # self.single_block.set_Xy(X=X, y=data) + # return self.single_block + + # def __iter__(self): + # self.files = iter(self.parts) + # return self + # + # def next(self): + # self.dataset.train = None + # self.dataset.valid = None + # self.dataset.test = None + # with open(self.data_dir + '/' + next(self.files), 'rb') as f: + # data = np.load(f) + # if self.dataset.preprocessor: + # logger.info('..applying preprocessing: ' + self.preprocessor.__class__.__name__) + # data = self.dataset.preprocessor.apply(data) + # + # if self.noise: + # logger.info('..applying noise: ' + self.noise.__class__.__name__) + # noisy_data = self.noise.apply(data) + # + # self.dataset.set_Xy(X=noisy_data, y=data) + # return self.dataset + # + # def nblocks(self): + # return len(self.parts) + +class Laura_Warp_Blocks(Laura_Root): + def __init__(self, data_dir=os.environ['PYNET_DATA_PATH'] + '/Laura_warp_npy', **kwargs): + parts = [ 'Laura_warp_data_000.npy', 'Laura_warp_data_010.npy', + 'Laura_warp_data_001.npy', 'Laura_warp_data_011.npy', + 'Laura_warp_data_002.npy', 'Laura_warp_data_012.npy', + 'Laura_warp_data_003.npy', 'Laura_warp_data_013.npy', + 'Laura_warp_data_004.npy', 'Laura_warp_data_014.npy', + 'Laura_warp_data_005.npy', 'Laura_warp_data_015.npy', + 'Laura_warp_data_006.npy', 'Laura_warp_data_016.npy', + 'Laura_warp_data_007.npy', 'Laura_warp_data_017.npy', + 'Laura_warp_data_008.npy', 'Laura_warp_data_018.npy', + 'Laura_warp_data_009.npy', 'Laura_warp_data_019.npy'] + data_paths = ["%s/%s"%(data_dir, part) for part in parts] + super(Laura_Warp_Blocks, self).__init__(data_paths=data_paths, **kwargs) + + +class Laura_Warp_Standardize_Blocks(Laura_Root): + def __init__(self, data_dir=os.environ['PYNET_DATA_PATH'] + '/Laura_warp_standardize_npy', **kwargs): + parts = [ 'Laura_warp_standardize_data_000.npy', 'Laura_warp_standardize_data_010.npy', 'Laura_warp_standardize_data_001.npy', 'Laura_warp_standardize_data_011.npy', 'Laura_warp_standardize_data_002.npy', 'Laura_warp_standardize_data_012.npy', 'Laura_warp_standardize_data_003.npy', 'Laura_warp_standardize_data_013.npy', @@ -95,13 +172,12 @@ def __init__(self, feature_size, target_size, **kwargs): 'Laura_warp_standardize_data_007.npy', 'Laura_warp_standardize_data_017.npy', 'Laura_warp_standardize_data_008.npy', 'Laura_warp_standardize_data_018.npy', 'Laura_warp_standardize_data_009.npy', 'Laura_warp_standardize_data_019.npy'] + data_paths = ["%s/%s"%(data_dir, part) for part in parts] + super(Laura_Warp_Standardize_Blocks, self).__init__(data_paths=data_paths, **kwargs) - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_warp_standardize_npy' - -class Laura_Standardize_Blocks(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Standardize_Blocks, self).__init__(feature_size, target_size, **kwargs) - self.parts = [ 'Laura_standardize_data_000.npy', 'Laura_standardize_data_010.npy', +class Laura_Standardize_Blocks(Laura_Root): + def __init__(self, data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_standardize_npy', **kwargs): + parts = [ 'Laura_standardize_data_000.npy', 'Laura_standardize_data_010.npy', 'Laura_standardize_data_001.npy', 'Laura_standardize_data_011.npy', 'Laura_standardize_data_002.npy', 'Laura_standardize_data_012.npy', 'Laura_standardize_data_003.npy', 'Laura_standardize_data_013.npy', @@ -111,126 +187,157 @@ def __init__(self, feature_size, target_size, **kwargs): 'Laura_standardize_data_007.npy', 'Laura_standardize_data_017.npy', 'Laura_standardize_data_008.npy', 'Laura_standardize_data_018.npy', 'Laura_standardize_data_009.npy', 'Laura_standardize_data_019.npy'] - - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_standardize_npy' + super(Laura_Standardize_Blocks, self).__init__(data_paths=data_paths, **kwargs) class Laura_Warp_Blocks_500_RELU(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_500_RELU, self).__init__(feature_size, target_size, **kwargs) - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0713_Warp_500_20140714_1317_43818059' + def __init__(self, **kwargs): + data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0713_Warp_500_20140714_1317_43818059' + super(Laura_Warp_Blocks_500_RELU, self).__init__(data_dir=data_dir, **kwargs) class Laura_Warp_Blocks_500_Tanh(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_500_Tanh, self).__init__(feature_size, target_size, **kwargs) - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0830_Warp_Blocks_2049_500_tanh_gpu_20140902_0012_36590657' + def __init__(self, **kwargs): + data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0830_Warp_Blocks_2049_500_tanh_gpu_20140902_0012_36590657' + super(Laura_Warp_Blocks_500_Tanh, self).__init__(data_dir=data_dir, **kwargs) class Laura_Warp_Blocks_500_Tanh_Dropout(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_500_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0916_Warp_Blocks_2049_500_tanh_tanh_gpu_dropout_20140916_1705_29139505' + super(Laura_Warp_Blocks_500_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + + +class Laura_Warp_Blocks_500_Tanh_Blackout(Laura_Warp_Blocks): + def __init__(self, **kwargs): + self.data_dir = os.environ['PYNET_DATA_PATH'] + '/AE1018_Warp_Blocks_2049_500_tanh_tanh_gpu_blackout_continue_20141018_1408_44747438' + super(Laura_Warp_Blocks_500_Tanh_Blackout, self).__init__(data_dir=data_dir, **kwargs) + + +class Laura_Warp_Blocks_180_Tanh_Blackout(Laura_Warp_Blocks): + def __init__(self, **kwargs): + self.data_dir = os.environ['PYNET_DATA_PATH'] + '/AE1018_Warp_Blocks_500_180_tanh_tanh_gpu_blackout_20141018_2238_56949300' + super(Laura_Warp_Blocks_180_Tanh_Blackout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_180_Tanh_Dropout(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_180_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0918_Warp_Blocks_2layers_finetune_2049_180_tanh_tanh_gpu_noisy_20140918_2113_17247388' + super(Laura_Warp_Blocks_180_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_180_Tanh(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_180_Tanh, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0914_Warp_Blocks_2layers_finetune_2049_180_gpu_20140915_0006_11454520' + super(Laura_Warp_Blocks_180_Tanh, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Scale_Warp_Blocks_500_Tanh(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Scale_Warp_Blocks_500_Tanh, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/AE0930_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_clean_20140930_1345_29800576' + super(Laura_Scale_Warp_Blocks_500_Tanh, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Scale_Warp_Blocks_180_Tanh(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Scale_Warp_Blocks_180_Tanh, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE1001_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_clean_20141002_0348_53679208' + super(Laura_Scale_Warp_Blocks_180_Tanh, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Scale_Warp_Blocks_500_Tanh_Dropout(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Scale_Warp_Blocks_500_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/AE1002_Scale_Warp_Blocks_2049_500_tanh_tanh_gpu_dropout_20141001_0321_33382955' + super(Laura_Scale_Warp_Blocks_500_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Scale_Warp_Blocks_180_Tanh_Dropout(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Scale_Warp_Blocks_180_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE1001_Scale_Warp_Blocks_500_180_tanh_tanh_gpu_dropout_20141001_2158_16765065' + super(Laura_Scale_Warp_Blocks_180_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_650(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_650, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0721_Warp_Blocks_650_20140722_2217_09001837' + super(Laura_Warp_Blocks_650, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_1000(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_1000, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0713_Warp_1000_20140714_1831_00043080' + super(Laura_Warp_Blocks_1000, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_250(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_250, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0717_warp_1000fea_20140717_1705_04859196' + super(Laura_Warp_Blocks_250, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_180(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_180, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0721_Warp_Blocks_500_180_20140723_0131_18179134' + super(Laura_Warp_Blocks_180, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Warp_Blocks_150(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Warp_Blocks_150, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0721_Warp_Blocks_180_150_20140723_1912_01578422' + super(Laura_Warp_Blocks_150, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Cut_Warp_Blocks_700(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Cut_Warp_Blocks_700, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/cut_0_700_laura_warp_npy' + super(Laura_Cut_Warp_Blocks_700, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Cut_Warp_Blocks_300(Laura_Warp_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Cut_Warp_Blocks_300, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_warp_cut_AE0730_Cut_Warp_Blocks_700_300_20140730_0134_17129588' + super(Laura_Cut_Warp_Blocks_300, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_500_Tanh_Tanh(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_500_Tanh_Tanh, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0912_Blocks_2049_500_tanh_tanh_gpu_20140914_1211_46292389' + super(Laura_Blocks_500_Tanh_Tanh, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_180_Tanh_Tanh(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_180_Tanh_Tanh, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0917_Blocks_2layers_finetune_2049_180_tanh_tanh_gpu_clean_20140917_1009_07286035' + super(Laura_Blocks_180_Tanh_Tanh, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_500_Tanh_Tanh_Dropout(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_500_Tanh_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0915_Blocks_2049_500_tanh_tanh_gpu_Dropout_20140915_1900_37160748' + super(Laura_Blocks_500_Tanh_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_180_Tanh_Tanh_Dropout(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_180_Tanh_Tanh_Dropout, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0917_Blocks_2layers_finetune_2049_180_tanh_tanh_gpu_noisy_20140917_1013_42539511' + super(Laura_Blocks_180_Tanh_Tanh_Dropout, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_500_Tanh_Sigmoid(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_500_Tanh_Sigmoid, self).__init__(feature_size, target_size, **kwargs) + def __init__(self, **kwargs): self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0915_Blocks_2049_500_tanh_sig_gpu_Dropout_20140915_1857_22433203' + super(Laura_Blocks_500_Tanh_Sigmoid, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_500(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_500, self).__init__(feature_size, target_size, **kwargs) - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0712_500_20140713_0345_22901754' + def __init__(self, **kwargs): + data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0712_500_20140713_0345_22901754' + super(Laura_Blocks_500, self).__init__(data_dir=data_dir, **kwargs) + class Laura_Blocks_1000(Laura_Blocks): - def __init__(self, feature_size, target_size, **kwargs): - super(Laura_Blocks_1000, self).__init__(feature_size, target_size, **kwargs) - self.data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0712_Warp_1000_20140712_1230_54443469' + def __init__(self, **kwargs): + data_dir = os.environ['PYNET_DATA_PATH'] + '/Laura_AE0712_Warp_1000_20140712_1230_54443469' + super(Laura_Blocks_1000, self).__init__(data_dir=data_dir, **kwargs) diff --git a/pynet/layer.py b/pynet/layer.py index d53a53d..364406f 100644 --- a/pynet/layer.py +++ b/pynet/layer.py @@ -11,7 +11,7 @@ class Layer(object): Abstract Class """ - def __init__(self, dim, name, W=None, b=None, dropout_below=None, blackout_below=None): + def __init__(self, dim, name, W=None, b=None, dropout_below=None, noise=None, blackout_below=None): """ DESCRIPTION: This is an abstract layer class @@ -20,7 +20,7 @@ def __init__(self, dim, name, W=None, b=None, dropout_below=None, blackout_below name(string): name of the layer W(tensor variable): Weight of 2D tensor matrix b(tensor variable): bias of 2D tensor matrix - dropout_below: probability of the inputs from the layer below been masked out + dropout_below(float): probability of the inputs from the layer below been masked out """ self.dim = dim self.name = name @@ -28,6 +28,7 @@ def __init__(self, dim, name, W=None, b=None, dropout_below=None, blackout_below self.b = b assert not (dropout_below and blackout_below), 'cannot set both dropout and blackout' self.dropout_below = dropout_below + self.noise = noise self.blackout_below = blackout_below if self.W is not None and self.W.name is None: @@ -43,8 +44,20 @@ def _test_fprop(self, state_below): def _train_fprop(self, state_below): raise NotImplementedError(str(type(self))+" does not implement _train_fprop.") + def _apply_noise(self, state_below): + """ + DESCRIPTION: + Adds noise to the layer during training + """ + if self.noise: + state_below = self.noise.apply(state_below) + return state_below - def _mask_state_below(self, state_below): + def _dropout_below(self, state_below): + """ + DESCRIPTION: + Applies dropout to the layer during training + """ if self.dropout_below is not None: assert self.dropout_below >= 0. and self.dropout_below <= 1., \ 'dropout_below is not in range [0,1]' @@ -52,14 +65,16 @@ def _mask_state_below(self, state_below): p=(1-self.dropout_below), dtype=floatX) * state_below return state_below - - def _blackout_below(self, state_below): - if self.blackout_below is not None: - assert self.blackout_below >= 0. and self.blackout_below <= 1., \ - 'blackout_below is not in range [0,1]' - state_below = theano_rand.binomial(size=(), n=1, p=(1-self.blackout_below), - dtype=floatX) * state_below - return state_below + # + # def _blackout_below(self, state_below): + # if self.blackout_below is not None: + # assert self.blackout_below >= 0. and self.blackout_below <= 1., \ + # 'blackout_below is not in range [0,1]' + # bin_rd = theano_rand.binomial(size=(state_below.shape[0],), n=1, p=(1-self.blackout_below), + # dtype=floatX) + # state_below = T.shape_padright(bin_rd) * state_below + # + # return state_below def _linear_part(self, state_below): """ @@ -70,7 +85,7 @@ def _linear_part(self, state_below): """ return T.dot(state_below, self.W) + self.b - def _dropout_test_fprop(self, state_below): + def _dropout_fprop(self, state_below): """ DESCRIPTION: resize the weight during testing for models trained with dropout. @@ -112,32 +127,33 @@ def _layer_stats(self, state_below, layer_output): # true_state = T.mean(T.gt(T.abs_(state_below), 0)).astype(floatX) # activity = T.mean(T.gt(layer_output, 0) * 1.0) # return [('activity', activity.astype(floatX)), - # # ('max_col_length', max_length), - # # ('mean_col_length', mean_length), - # # ('min_col_length', min_length), - # ('output_max', max_output), - # ('output_mean', mean_output), - # ('output_min', min_output)] - # ('pos', pos), - # ('test', out)] - # ('max_W', T.max(self.W)), - # ('mean_W', T.mean(self.W)), - # ('min_W', T.min(self.W)), - # ('max_b', T.max(self.b)), - # ('mean_b', T.mean(self.b)), - # ('min_b', T.min(self.b))] + # ('max_col_length', max_length), + # ('mean_col_length', mean_length), + # ('min_col_length', min_length), + # ('output_max', max_output), + # ('output_mean', mean_output), + # ('output_min', min_output)] + # ('pos', pos), + # ('test', out)] + # ('max_W', T.max(self.W)), + # ('mean_W', T.mean(self.W)), + # ('min_W', T.min(self.W)), + # ('max_b', T.max(self.b)), + # ('mean_b', T.mean(self.b)), + # ('min_b', T.min(self.b))] return [] class Linear(Layer): def _test_fprop(self, state_below): if self.dropout_below: - return self._dropout_test_fprop(state_below) + return self._dropout_fprop(state_below) else: return self._linear_part(state_below) def _train_fprop(self, state_below): - state_below = self._mask_state_below(state_below) - state_below = self._blackout_below(state_below) + state_below = self._dropout_below(state_below) + # state_below = self._blackout_below(state_below) + state_below = self._apply_noise(state_below) output = self._linear_part(state_below) return output diff --git a/pynet/layer_noise.py b/pynet/layer_noise.py new file mode 100644 index 0000000..9c0889f --- /dev/null +++ b/pynet/layer_noise.py @@ -0,0 +1,75 @@ +""" +Functionality : Define the noise that is to be added to each layer +""" + +import theano +import theano.tensor as T +from theano.tensor.shared_randomstreams import RandomStreams + +floatX = theano.config.floatX +theano_rand = RandomStreams(seed=1012) + +class Noise(object): + """ + This is an abstract class for applying noise to each layer + """ + + def apply(self, X): + """ + DESCRIPTION: + This method applies noise to X and return a noisy X + PARAM: + X : 2d numpy array of dimension number of examples by number of dimensions + """ + raise NotImplementedError(str(type(self))+" does not implement an apply method.") + + + +class MaskOut(Noise): + + """ + This noise masked out a portion of the dimension from each example + """ + + def __init__(self, ratio=0.5): + """ + PARAM: + ratio : float + The portion of the inputs that is masked out + """ + self.ratio = ratio + + def apply(self, X): + return X * theano_rand.binomial(size=X.shape, n=1, p=(1-self.ratio), dtype=floatX) + + +class Gaussian(Noise): + """ + Applies gaussian noise to each value of X + """ + + def __init__(self, std=0.1, mean=0): + self.std = std + self.mean = mean + + def apply(self, X): + return X + theano_rand.normal(avg=self.mean, std=self.std, size=X.shape, dtype=floatX) + + +class BlackOut(Noise): + """ + This noise masked out a random example in a dataset, + adding noise in the time dimension + """ + + def __init__(self, ratio=0.5): + """ + PARAM: + ratio : float + The portion of the examples that is masked out + """ + self.ratio = ratio + + def apply(self, X): + rd = theano_rand.binomial(size=(X.shape[0],), n=1, p=(1-self.ratio), dtype=floatX) + return X * T.shape_padright(rd) diff --git a/pynet/log.py b/pynet/log.py index eb95fb3..5726ef6 100644 --- a/pynet/log.py +++ b/pynet/log.py @@ -55,9 +55,9 @@ def __init__(self, experiment_name="experiment", description=None, if save_epoch_error: self.epoch_error_tbl = {"epoch":[], - "train_error":[], - "valid_error":[], - "test_error":[]} + "train_error":[], + "valid_error":[], + "test_error":[]} self.logger.info('exp_id: ' + self.exp_id) diff --git a/pynet/model.py b/pynet/model.py index 25e8bd8..931d341 100644 --- a/pynet/model.py +++ b/pynet/model.py @@ -47,10 +47,6 @@ def add_layer(self, layer): high = 4 * np.sqrt(6. / (layer.dim + prev_layer_dim)), size = (prev_layer_dim, layer.dim)), dtype = floatX) - # W_values = np.asarray(np.random.uniform(low = 0, - # high = 4 * np.sqrt(6. / (layer.dim + prev_layer_dim)), - # size = (prev_layer_dim, layer.dim)), - # dtype = floatX) layer.W = theano.shared(value=W_values, name='W_'+layer.name, borrow=True) @@ -64,7 +60,6 @@ def pop_layer(self, index): return self.layers.pop(index) def test_fprop(self, input_state): - test_layers_stats = [] for i in xrange(len(self.layers)): layer_output = self.layers[i]._test_fprop(input_state) @@ -78,7 +73,6 @@ def test_fprop(self, input_state): def train_fprop(self, input_state): - train_layers_stats = [] for i in xrange(len(self.layers)): layer_output = self.layers[i]._train_fprop(input_state) diff --git a/pynet/train_object.py b/pynet/train_object.py index 69b5e7d..9687f9b 100644 --- a/pynet/train_object.py +++ b/pynet/train_object.py @@ -12,6 +12,7 @@ logging.basicConfig(level=logging.DEBUG) from pynet.log import Log + from pynet.utils.utils import split_list, generate_shared_list, \ merge_lists, get_shared_values, \ duplicate_param @@ -352,7 +353,7 @@ def run(self): self.log.experiment_name)) if self.log.save_epoch_error: - self.log._save_epoch_error(best_epoch, best_train_error, best_valid_error, best_test_error) + self.log._save_epoch_error(epoch, mean_train_error, mean_valid_error, mean_test_error) self.log.info('..epoch error saved') end_time = time.time() diff --git a/scripts/synthesis.sh b/scripts/synthesis.sh index 2a87385..30ae582 100755 --- a/scripts/synthesis.sh +++ b/scripts/synthesis.sh @@ -1,12 +1,16 @@ #!/bin/bash -VCTK_HOME= +VCTK_HOME=/Volumes/Storage +# VCTK_HOME=/home/smg/zhenzhou + SPEC_DIR= SPEC_EXT= WAV_DIR= files= dtype= + + while [ "$1" != "" ]; do case $1 in --vctk_home) shift @@ -41,8 +45,7 @@ while [ "$1" != "" ]; do done -# VCTK_HOME=/Volumes/Storage -VCTK_HOME=/home/smg/zhenzhou + . $VCTK_HOME/VCTK/Research-Demo/fa-tts/STRAIGHT-TTS/local.conf.0 TMP_DIR=$VCTK_HOME/VCTK/Research-Demo/fa-tts/STRAIGHT-TTS/tmp/England/Laura