Merged with master branch

Author: lakshaykc

scripts/batch_generator.py

@@ -62,13 +62,14 @@ def get_configs():
     configs.DEFINE_integer("min_years",None,"Alt to min_unrollings")
     configs.DEFINE_integer("max_years",None,"Alt to max_unrollings")
     configs.DEFINE_integer("pls_years",None,"Alt to max_years. max_years = min_year+pls_years")
-    # num_unrollings is being depricated by max_unrollings
+    # num_unrollings is being depricated, replaced with max_unrollings
     configs.DEFINE_integer("num_unrollings",4,"Number of unrolling steps")
     configs.DEFINE_integer("stride",12,"How many steps to skip per unrolling")
     configs.DEFINE_integer("forecast_n",12,"How many steps to forecast into the future")
     configs.DEFINE_integer("batch_size",1,"Size of each batch")
     configs.DEFINE_integer("num_layers",1, "Numer of RNN layers")
     configs.DEFINE_integer("num_hidden",10,"Number of hidden layer units")
+    configs.DEFINE_float("training_noise",None, "Level of training noise as multiple of 1-stdev")
     configs.DEFINE_float("init_scale",0.1, "Initial scale for weights")
     configs.DEFINE_float("max_grad_norm",10.0,"Gradient clipping")
     configs.DEFINE_integer("start_date",None,"First date to train on as YYYYMM")
@@ -102,6 +103,7 @@ def get_configs():
     configs.DEFINE_float("passes",1.0,"Passes through day per epoch")
     configs.DEFINE_float("target_lambda",0.5,"How much to weight last step vs. all steps in loss")
     configs.DEFINE_float("rnn_lambda",0.5,"How much to weight last step vs. all steps in loss")
+    configs.DEFINE_float("l2_alpha",0.0,"L2 Regularization")
     configs.DEFINE_integer("max_epoch",0,"Stop after max_epochs")
     configs.DEFINE_integer("early_stop",None,"Early stop parameter")
     configs.DEFINE_integer("seed",None,"Seed for deterministic training")

scripts/deep_quant.py

@@ -108,9 +108,6 @@ def debug_step(self, sess, batch, training=False, uq=False, UQ_model_type='MVE')
         np.set_printoptions(suppress=True)
         np.set_printoptions(precision=3)
 
-        print()
-        print(batch.inputs[-1][0][18:22])
-
         feed_dict = self._get_feed_dict(batch,keep_prob=1.0,training=training)
 
         # (s,t,lt,lkt,lkti,o,lo,lko,lkoi) = sess.run([self._seq_lengths,self._t,self._lt,self._lkt,self._lkti,self._o,self._lo,self._lko,self._lkoi],feed_dict)

scripts/models/base_model.py

@@ -167,10 +167,17 @@ def rnn_cell():
         # here is the learning part of the graph
         p1 = config.target_lambda
         p2 = config.rnn_lambda
-        l2 = config.l2_alpha*sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables() if "_b" not in tf_var.name)
-        loss = p1 * self._mse_0 + (1.0-p1)*(p2*self._mse_1 + (1.0-p2)*self._mse_2) + l2
+
         tvars = tf.trainable_variables()
-        grads = tf.gradients(loss, tvars)
+
+        l2 = config.l2_alpha * sum(
+            tf.nn.l2_loss(tf_var)
+            for tf_var in tvars
+            if not ("_b" in tf_var.name)
+        )
+
+        loss = p1 * self._mse_0 + (1.0-p1)*(p2*self._mse_1 + (1.0-p2)*self._mse_2) + l2
+        grads = tf.gradients(loss,tvars)
 
         if (config.max_grad_norm > 0):
             grads, self._grad_norm = tf.clip_by_global_norm(grads,config.max_grad_norm)

scripts/models/deep_rnn_model.py

@@ -0,0 +1,43 @@
+# Copyright 2016 Euclidean Technologies Management LLC  All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import os
+import time
+import sys
+import random
+
+import numpy as np
+import pandas as pd
+import math
+import copy
+
+class NoiseModel(object):
+    def __init__(self, seed=None, scaling_params=None, degree=0.00):
+        self._scale = scaling_params['scale']
+        self._degree = degree
+        if seed is not None:
+            np.random.seed(seed)
+
+    def add_noise(self,batch):
+        batch = copy.deepcopy(batch)
+        inputs = batch.inputs
+        input_scales = np.tile(self._scale,(batch.size,1))
+        num_inputs = inputs[0].shape[1]
+        for i in range(len(inputs)):
+            input_noise = np.random.normal(loc=0.0,
+                                           scale=self._degree,
+                                           size=(batch.size,num_inputs))
+            batch.inputs[i] += input_scales * input_noise
+        return batch

scripts/noise_model.py

@@ -24,16 +24,19 @@
 import tensorflow as tf
 import regex as re
 import math
+import numpy as np
 
 from utils import data_utils, model_utils
+from noise_model import NoiseModel
 
 def pretty_progress(step, prog_int, dot_count):
     if ( (prog_int<=1) or (step % (int(prog_int)+1)) == 0):
         dot_count += 1; print('.',end=''); sys.stdout.flush()
     return dot_count
 
 def run_epoch(session, model, train_data, valid_data,
-                keep_prob=1.0, passes=1.0, verbose=False):
+              keep_prob=1.0, passes=1.0, 
+              noise_model=None, verbose=False):
 
     if not train_data.num_batches > 0:
         raise RuntimeError("batch_size*max_unrollings is larger "
@@ -54,17 +57,18 @@ def run_epoch(session, model, train_data, valid_data,
 
     for step in range(train_steps):
         batch = train_data.next_batch()
+        if noise_model is not None:
+            batch = noise_model.add_noise(batch)
         train_mse += model.train_step(session, batch, keep_prob=keep_prob)
         if verbose: dot_count = pretty_progress(step,prog_int,dot_count)
 
+    # evaluate validation data
     for step in range(valid_steps):
         batch = valid_data.next_batch()
         (mse,_) = model.step(session, batch)
         valid_mse += mse
         if verbose: dot_count = pretty_progress(train_steps+step,prog_int,dot_count)
 
-    # evaluate validation data
-
     if verbose:
         print("."*(100-dot_count),end='')
         print(" passes: %.2f  "
@@ -117,19 +121,15 @@ def train_model(config):
         print("Constructing model ...")
         model = model_utils.get_model(session, config, verbose=True)
 
-        if config.data_scaler is not None:
-            start_time = time.time()
-            print("Calculating scaling parameters ...", end=' '); sys.stdout.flush()
-            scaling_params = train_data.get_scaling_params(config.data_scaler)
-            model.set_scaling_params(session,**scaling_params)
-            print("done in %.2f seconds."%(time.time() - start_time))
-            print("%-10s %-6s %-6s"%('feature','mean','std'))
-            for i in range(len(train_data.feature_names)):
-                center = "%.4f"%scaling_params['center'][i];
-                scale  = "%.4f"%scaling_params['scale'][i];
-                print("%-10s %-6s %-6s"%(train_data.feature_names[i],
-                                         center,scale))
-            sys.stdout.flush()
+        params = model_utils.get_scaling_params(config,train_data,verbose=True)
+        model.set_scaling_params(session,**params)
+
+        noise_model = None
+        if config.training_noise is not None:
+            print("Training noise level: %.2f * 1-stdev"%config.training_noise) 
+            noise_model = NoiseModel(seed=config.seed,
+                                     scaling_params=params,
+                                     degree=config.training_noise)
 
         if config.early_stop is not None:
             print("Training will early stop without "
@@ -148,6 +148,7 @@ def train_model(config):
             (train_mse, valid_mse) = run_epoch(session, model, train_data, valid_data,
                                                keep_prob=config.keep_prob, 
                                                passes=config.passes,
+                                               noise_model=noise_model,
                                                verbose=True)
             print( ('Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning rate: %.4f') %
                   (i + 1, train_mse, valid_mse, lr) )

scripts/train.py

@@ -70,6 +70,31 @@ def adjust_learning_rate(session, model,
     model.set_learning_rate(session, learning_rate)
     return learning_rate
 
+def get_scaling_params(config, data, verbose=False):
+    # Initialize scaling params
+    scaling_params = None
+    if config.scalesfile is not None and os.path.isfile(config.scalesfile):
+        scaling_params = pickle.load( open( config.scalesfile, "rb" ) )
+        if verbose:
+            print("Reading scaling params from %s"%config.scalesfile);
+    else:
+        scaling_params = data.get_scaling_params(config.data_scaler)
+        if config.scalesfile is not None:
+            pickle.dump(scaling_params, open( config.scalesfile, "wb" ))
+            if verbose:
+                print("Writing scaling params to %s"%config.scalesfile);
+
+    if verbose:
+        print("Scaling params are:")
+        print("%-10s %-6s %-6s"%('feature','mean','std'))
+        for i in range(len(data.feature_names)):
+            center = "%.4f"%scaling_params['center'][i];
+            scale  = "%.4f"%scaling_params['scale'][i];
+            print("%-10s %-6s %-6s"%(data.feature_names[i],
+                                     center,scale))
+    return scaling_params
+
+
 def get_model(session, config, verbose=False):
     """
     Args:
@@ -79,30 +104,25 @@ def get_model(session, config, verbose=False):
     Returns:
       the model
     """
-    if config.nn_type == 'logreg':
-        model_file = os.path.join(config.model_dir, "logreg.pkl" )
-        clf = LogRegModel(load_from=model_file)
-        mtrain, mdeploy = clf, clf
-
+    
+    model = _create_model(session, config, verbose)
+
+    ckpt = tf.train.get_checkpoint_state(config.model_dir)
+    start_time = time.time()
+    if ckpt and gfile.Exists(ckpt.model_checkpoint_path+".index"):
+        if verbose:
+            print("Reading model parameters from {}...".format(
+                ckpt.model_checkpoint_path), end=' ')
+        tf.train.Saver(max_to_keep=200).restore(session,
+                                                ckpt.model_checkpoint_path)
+        if verbose:
+            print("done in %.2f seconds."%(time.time() - start_time))
     else:
-        model = _create_model(session, config, verbose)
-
-        ckpt = tf.train.get_checkpoint_state(config.model_dir)
-        start_time = time.time()
-        if ckpt and gfile.Exists(ckpt.model_checkpoint_path+".index"):
-            if verbose:
-                print("Reading model parameters from {}...".format(
-                    ckpt.model_checkpoint_path), end=' ')
-            tf.train.Saver(max_to_keep=200).restore(session,
-                                                    ckpt.model_checkpoint_path)
-            if verbose:
-                print("done in %.2f seconds."%(time.time() - start_time))
-        else:
-            if verbose:
-                print("Creating model with fresh parameters ...", end=' ')
-            session.run(tf.global_variables_initializer())
-            if verbose:
-                print("done in %.2f seconds."%(time.time() - start_time))
+        if verbose:
+            print("Creating model with fresh parameters ...", end=' ')
+        session.run(tf.global_variables_initializer())
+        if verbose:
+            print("done in %.2f seconds."%(time.time() - start_time))
 
     return model